diff --git a/auto.sh b/auto.sh
index b213007..315b81c 100755
--- a/auto.sh
+++ b/auto.sh
@@ -16,7 +16,7 @@
## --with-openssl-opt="enable-weak-ssl-ciphers -DCFLAGS='-march=native -O3 -fuse-linker-plugin'" \
### Remove Old file
-rm -rf /usr/sbin/nginx.old
+rm -f /usr/sbin/nginx.old
### Multithread build
BUILD_MTS="-j$(expr $(nproc) \+ 1)"
@@ -38,16 +38,27 @@ if [ ! -f "lib/zlib/Makefile" ]; then
fi
### PSOL Download (PageSpeed)
-if [ ! -d "lib/ngx_pagespeed/psol" ]; then
- cd lib/ngx_pagespeed
- curl "$(scripts/format_binary_url.sh PSOL_BINARY_URL)" | tar xz
- cd ../../
+if [ ! -d "lib/pagespeed" ]; then
+ ### Download pagespeed
+ cd lib
+ wget -c https://github.com/apache/incubator-pagespeed-ngx/archive/v1.13.35.2-stable.zip
+ unzip v1.13.35.2-stable.zip
+ rm -f v1.13.35.2-stable.zip
+ cd incubator-pagespeed-ngx-1.13.35.2-stable
+
+ ### Download psol
+ wget -c https://dl.google.com/dl/page-speed/psol/1.13.35.2-x64.tar.gz
+ tar -xzvf 1.13.35.2-x64.tar.gz
+ rm -f 1.13.35.2-x64.tar.gz
+ cd ..
+ mv incubator-pagespeed-ngx-1.13.35.2-stable pagespeed
+ cd ..
fi
auto/configure \
--with-cc-opt='-DTCP_FASTOPEN=23 -m64 -flto -g -O3 -march=native -fstack-protector-strong -fuse-ld=gold -fuse-linker-plugin --param=ssp-buffer-size=4 -Wformat -Werror=format-security -Wno-strict-aliasing -Wp,-D_FORTIFY_SOURCE=2 -gsplit-dwarf -DNGX_HTTP_HEADERS' \
---with-ld-opt='-ljemalloc -Wl,-z,relro' \
---with-openssl-opt="enable-tls13downgrade enable-ec_nistp_64_gcc_128 enable-weak-ssl-ciphers no-ssl3-method" \
+--with-ld-opt='-lrt -ljemalloc -Wl,-z,relro -Wl,-z,now -fPIC' \
+--with-openssl-opt="enable-tls13downgrade enable-ec_nistp_64_gcc_128 enable-weak-ssl-ciphers no-ssl3-method -DCFLAGS='-O3 -march=native -fuse-linker-plugin -ljemalloc'" \
--builddir=objs --prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \
--pid-path=/var/run/nginx.pid \
@@ -98,7 +109,7 @@ auto/configure \
--with-stream_ssl_preread_module \
--add-module=./lib/ngx_devel_kit \
--add-module=./lib/ngx_brotli \
---add-module=./lib/ngx_pagespeed ${PS_NGX_EXTRA_FLAGS} \
+--add-module=./lib/pagespeed ${PS_NGX_EXTRA_FLAGS} \
--add-module=./lib/ngx-fancyindex \
--add-module=./lib/naxsi/naxsi_src \
--add-module=./lib/nginx-dav-ext-module \
diff --git a/lib/ngx_pagespeed/.gitignore b/lib/ngx_pagespeed/.gitignore
deleted file mode 100644
index f03b88d..0000000
--- a/lib/ngx_pagespeed/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-test/tmp
-psol/
-psol-*.tar.gz
-*.*.*.*.tar.gz
-nginx
diff --git a/lib/ngx_pagespeed/.gitmodules b/lib/ngx_pagespeed/.gitmodules
deleted file mode 100644
index e69de29..0000000
diff --git a/lib/ngx_pagespeed/.travis.yml b/lib/ngx_pagespeed/.travis.yml
deleted file mode 100644
index 667bcab..0000000
--- a/lib/ngx_pagespeed/.travis.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-language: c++
-env:
- global:
- - MAKEFLAGS=-j3
-# By default travis loads submodules serially, but we can load them in parallel
-# if we install an updated git and use --jobs. Some timing numbers:
-# serial: 257s
-# jobs=4: 182s (29s to install new git, 153s to run the downloads)
-# jobs=8: 179s (29s to install new git, 150s to run the downloads)
-# We can't use --depth=1, though, because github doesn't have
-# allowReachableSHA1InWant set.
-#
-# TODO(jefftk): once we're running on a server with git 2.8 or later we can have
-# --jobs without installing a new git.
-git:
- submodules: false
-before_install:
- - sudo add-apt-repository --yes ppa:git-core/ppa
- - sudo apt-get update
- - sudo apt-get install git
- - git submodule update --init --recursive --jobs=8
-install:
- scripts/build_ngx_pagespeed.sh --devel --assume-yes
-script:
- test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \
- $PWD/nginx/sbin/nginx
-dist: trusty
-sudo: required
-compiler:
- - gcc
-notifications:
- email:
- - pagespeed-ci@googlegroups.com
diff --git a/lib/ngx_pagespeed/LICENSE b/lib/ngx_pagespeed/LICENSE
deleted file mode 100644
index 9d18d4c..0000000
--- a/lib/ngx_pagespeed/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright 2010 Google Inc.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/lib/ngx_pagespeed/PSOL_BINARY_URL b/lib/ngx_pagespeed/PSOL_BINARY_URL
deleted file mode 100644
index 02bc497..0000000
--- a/lib/ngx_pagespeed/PSOL_BINARY_URL
+++ /dev/null
@@ -1 +0,0 @@
-https://dl.google.com/dl/page-speed/psol/1.13.35.2-$BIT_SIZE_NAME.tar.gz
diff --git a/lib/ngx_pagespeed/README.md b/lib/ngx_pagespeed/README.md
deleted file mode 100644
index 364cc91..0000000
--- a/lib/ngx_pagespeed/README.md
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-[](https://travis-ci.org/pagespeed/ngx_pagespeed)
-
-ngx_pagespeed speeds up your site and reduces page load time by automatically
-applying web performance best practices to pages and associated assets (CSS,
-JavaScript, images) without requiring you to modify your existing content or
-workflow. Features include:
-
-- Image optimization: stripping meta-data, dynamic resizing, recompression
-- CSS & JavaScript minification, concatenation, inlining, and outlining
-- Small resource inlining
-- Deferring image and JavaScript loading
-- HTML rewriting
-- Cache lifetime extension
-- and
- [more](https://developers.google.com/speed/docs/mod_pagespeed/config_filters)
-
-To see ngx_pagespeed in action, with example pages for each of the
-optimizations, see our demonstration site.
-
-## How to build
-
-Follow the steps on build
-ngx_pagespeed from source.
-
-## How to use
-
-Follow the steps on PageSpeed
-configuration.
-
-For feedback, questions, and to follow
-the progress of the project:
-
-- [ngx-pagespeed-discuss mailing
- list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
-- [ngx-pagespeed-announce mailing
- list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
diff --git a/lib/ngx_pagespeed/config b/lib/ngx_pagespeed/config
deleted file mode 100644
index aeb677b..0000000
--- a/lib/ngx_pagespeed/config
+++ /dev/null
@@ -1,390 +0,0 @@
-# Copyright 2012 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Environment Variables (Optional):
-# MOD_PAGESPEED_DIR: absolute path to the mod_pagespeed/src directory
-# PSOL_BINARY: absolute path to pagespeed_automatic.a
-# PSOL_BUILDTYPE: Release or Debug
-
-mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
-position_aux="${POSITION_AUX:-unset}"
-psol_buildtype="${PSOL_BUILDTYPE:-unset}"
-
-if [ "$mod_pagespeed_dir" = "unset" ] ; then
- mod_pagespeed_dir="$ngx_addon_dir/psol/include"
- build_from_source=false
-
- if [ ! -e "$mod_pagespeed_dir" ] ; then
- echo "ngx_pagespeed: pagespeed optimization library not found:"
-
- psol_binary_url="$($ngx_addon_dir/scripts/format_binary_url.sh \
- $ngx_addon_dir/PSOL_BINARY_URL)"
- if [[ "$psol_binary_url" != https://* ]]; then
- echo "
- This is a development branch of ngx_pagespeed, which means there is no
- precompiled PSOL library available to link against. Either build from a
- release tag, like latest-beta, or build PSOL from source:
- https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source"
- exit 1
- fi
-
- echo "
- You need to separately download the pagespeed library:
- $ cd $ngx_addon_dir
- $ wget $psol_binary_url
- $ tar -xzvf $(basename $psol_binary_url) # expands to psol/
-
- Or see the installation instructions:
- https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source"
- exit 1
- fi
-else
- build_from_source=true
-fi
-
-os_name='unknown_os'
-arch_name='unknown_arch'
-uname_os=`uname`
-uname_arch=`uname -m`
-
-if [ $uname_os = 'Linux' ]; then
- os_name='linux'
-elif [ $uname_os = 'Darwin' ]; then
- os_name='mac'
-else
- echo "OS not supported: $uname_os"
- exit 1
-fi
-
-if [ $uname_arch = 'x86_64' -o $uname_arch = 'amd64' ]; then
- arch_name='x64'
-elif [ $uname_arch = 'x86_32' -o $uname_arch = 'i686' \
- -o $uname_arch = 'i386' ]; then
- arch_name='ia32'
-else
- echo "Architecture not supported: $uname_arch"
- exit 1
-fi
-
-if [ "$psol_buildtype" = "unset" ] ; then
- if [ "$NGX_DEBUG" = "YES" ]; then
- buildtype=Debug
- # If we're using a psol tarball that doesn't contain Debug/ (which is the case
- # from 1.12 onward) then this will be overriden to buildtype=Release below.
- else
- buildtype=Release
- fi
-else
- buildtype=$psol_buildtype
-fi
-
-# If the compiler is gcc, we want to use g++ to link, if at all possible,
-# so that -static-libstdc++ works.
-# Annoyingly, the feature test doesn't even use $LINK for linking, so that
-# needs an explicit -lstdc++
-pagespeed_libs=
-ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
-ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
-if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
- LINK=$ps_maybe_gpp
-fi
-pagespeed_libs="-lstdc++"
-
-# The compiler needs to know that __sync_add_and_fetch_4 is ok,
-# and this requires an instruction that didn't exist on i586 or i386.
-if [ "$uname_arch" = "i686" ]; then
- FLAG_MARCH='-march=i686'
-fi
-
-CFLAGS="$CFLAGS $FLAG_MARCH"
-
-# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
-# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
-# to our libpsol.a
-# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
-CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
-CC_OLD_TEST_FLAGS="$CC_TEST_FLAGS"
-CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0 --std=c++11"
-
-case "$NGX_GCC_VER" in
- 4.8*)
- # On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs. This breaks
- # on VerifySizesAreEqual in bit_cast in chromium/src/base/basictypes.h which
- # has a typedef that is intentionally unused.
- CFLAGS="$CFLAGS -Wno-unused-local-typedefs"
-
- # On GCC 4.8 and above, we get the following compiler warning:
- # chromium/src/base/memory/scoped_ptr.h:133:7: warning: declaration of ‘class scoped_ptr’ [enabled by default]
- # Based on discussion at http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54055,
- # this is invalid code, but hasn't been fixed yet in chromium.
- # Unfortunately, there also does not appear to be a flag for just disabling
- # that warning, so we add Wno-error to override nginx's default -Werror
- # option.
- CFLAGS="$CFLAGS -Wno-error"
- ;;
-esac
-
-# workaround for a bug in nginx-1.9.11, see:
-# http://hg.nginx.org/nginx/rev/ff1e625ae55b
-NGX_VERSION=`grep nginx_version src/core/nginx.h | sed -e 's/^.* \(.*\)$/\1/'`
-if [ "$NGX_VERSION" = "1009011" ]; then
- CFLAGS="$CFLAGS -Wno-write-strings"
-fi
-
-if [ "$WNO_ERROR" = "YES" ]; then
- CFLAGS="$CFLAGS -Wno-error"
-fi
-
-psol_binary="${PSOL_BINARY:-unset}"
-if [ "$psol_binary" = "unset" ] ; then
- if $build_from_source ; then
- psol_binary="\
- $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
- else
- if ! [ -d "$ngx_addon_dir/psol/lib/$buildtype" ]; then
- echo "
-You have set --with-debug for building nginx, but precompiled Debug binaries for
-PSOL, which ngx_pagespeed depends on, aren't available. If you're trying to
-debug PSOL you need to build it from source. If you just want to run nginx with
-debug-level logging you can use the Release binaries."
- echo -n "
-Use the available Release binaries?"
- read -p " [Y/n] " yn
- if [[ "$yn" == N* || "$yn" == n* ]]; then
- echo "Cancelled."
- exit 1
- fi
-
- buildtype=Release
- fi
- psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
- psol_binary="$psol_library_dir/pagespeed_automatic.a"
- fi
-fi
-
-echo "mod_pagespeed_dir=$mod_pagespeed_dir"
-echo "build_from_source=$build_from_source"
-
-ngx_feature="psol"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs="
-#include \"pagespeed/kernel/base/string.h\"
-#include \"pagespeed/kernel/base/string_writer.h\"
-#include \"pagespeed/kernel/base/null_message_handler.h\"
-#include \"pagespeed/kernel/html/html_parse.h\"
-#include \"pagespeed/kernel/html/html_writer_filter.h\"
-"
-
-pagespeed_include="\
- $mod_pagespeed_dir \
- $mod_pagespeed_dir/third_party/chromium/src \
- $mod_pagespeed_dir/third_party/google-sparsehash/src/src \
- $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
- $mod_pagespeed_dir/third_party/grpc/src/include \
- $mod_pagespeed_dir/third_party/protobuf/src/src \
- $mod_pagespeed_dir/third_party/re2/src \
- $mod_pagespeed_dir/out/$buildtype/obj/gen \
- $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
- $mod_pagespeed_dir/third_party/apr/src/include \
- $mod_pagespeed_dir/third_party/aprutil/src/include \
- $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
- $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
- $mod_pagespeed_dir/url"
-ngx_feature_path="$pagespeed_include"
-
-pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm -luuid"
-ngx_feature_libs="$pagespeed_libs"
-ngx_feature_test="
-
- GoogleString output_buffer;
- net_instaweb::StringWriter write_to_string(&output_buffer);
-
- net_instaweb::NullMessageHandler handler;
- net_instaweb::HtmlParse html_parse(&handler);
- net_instaweb::HtmlWriterFilter html_writer_filter(&html_parse);
-
- html_writer_filter.set_writer(&write_to_string);
- html_parse.AddFilter(&html_writer_filter);
-
- html_parse.StartParse(\"http:example.com\");
- html_parse.ParseText(
- \"Test
Test Text
\n\");
- html_parse.FinishParse();
-
- printf(\"parsed as: %s\", output_buffer.c_str())"
-
-# Test whether we have pagespeed and can compile and link against it.
-. "$ngx_addon_dir/cpp_feature"
-
-if [ $ngx_found = no ]; then
- cat << END
-$0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in $PWD/$NGX_AUTOCONF_ERR for more details.
-END
- exit 1
-fi
-
-ps_src="$ngx_addon_dir/src"
-ngx_addon_name=ngx_pagespeed
-NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-$ps_src/log_message_handler.h \
-$ps_src/ngx_base_fetch.h \
-$ps_src/ngx_caching_headers.h \
-$ps_src/ngx_event_connection.h \
-$ps_src/ngx_fetch.h \
-$ps_src/ngx_gzip_setter.h \
-$ps_src/ngx_list_iterator.h \
-$ps_src/ngx_message_handler.h \
-$ps_src/ngx_pagespeed.h \
-$ps_src/ngx_rewrite_driver_factory.h \
-$ps_src/ngx_rewrite_options.h \
-$ps_src/ngx_server_context.h \
-$ps_src/ngx_url_async_fetcher.h \
-$psol_binary"
-NPS_SRCS=" \
-$ps_src/log_message_handler.cc \
-$ps_src/ngx_base_fetch.cc \
-$ps_src/ngx_caching_headers.cc \
-$ps_src/ngx_event_connection.cc \
-$ps_src/ngx_fetch.cc \
-$ps_src/ngx_gzip_setter.cc \
-$ps_src/ngx_list_iterator.cc \
-$ps_src/ngx_message_handler.cc \
-$ps_src/ngx_pagespeed.cc \
-$ps_src/ngx_rewrite_driver_factory.cc \
-$ps_src/ngx_rewrite_options.cc \
-$ps_src/ngx_server_context.cc \
-$ps_src/ngx_url_async_fetcher.cc"
-# Save our sources in a separate var since we may need it in config.make
-PS_NGX_SRCS="$NGX_ADDON_SRCS \
-$NPS_SRCS"
-
-# Make pagespeed run immediately before gzip and Brotli.
-if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
- next=ngx_http_brotli_filter_module
-elif [ $HTTP_GZIP = YES ]; then
- next=ngx_http_gzip_filter_module
-else
- next=ngx_http_range_header_filter_module
-fi
-
-if [ -n "$ngx_module_link" ]; then
- # nginx-1.9.11+
- ngx_module_type=HTTP_FILTER
- ngx_module_name="ngx_pagespeed ngx_pagespeed_etag_filter"
- ngx_module_incs="$ngx_feature_path"
- ngx_module_deps=
- ngx_module_srcs="$NPS_SRCS"
- ngx_module_libs="$ngx_feature_libs"
- ngx_module_order="ngx_http_range_header_filter_module\
- ngx_pagespeed_etag_filter\
- ngx_http_gzip_filter_module \
- ngx_http_brotli_filter_module \
- ngx_pagespeed \
- ngx_http_postpone_filter_module \
- ngx_http_ssi_filter_module \
- ngx_http_charset_filter_module \
- ngx_http_xslt_filter_module \
- ngx_http_image_filter_module \
- ngx_http_sub_filter_module \
- ngx_http_addition_filter_module \
- ngx_http_gunzip_filter_module \
- ngx_http_userid_filter_module \
- ngx_http_headers_filter_module"
-
- . auto/module
-
- if [ $ngx_module_link != DYNAMIC ]; then
- # ngx_module_order doesn't work with static modules,
- # so we must re-order filters here.
- if [ "$position_aux" = "true" ] ; then
- HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
- else
- HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
- | sed "s/ngx_pagespeed//" \
- | sed "s/$next/$next ngx_pagespeed/")
- fi
- # Make the etag header filter run immediately before range header filter.
- HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES \
- | sed "s/ngx_pagespeed_etag_filter//" \
- | sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
- else
- if [ "$position_aux" = "true" ] ; then
- ngx_module_type=HTTP_AUX_FILTER
- ngx_module_order=""
- fi
- fi
-else
- CORE_LIBS="$CORE_LIBS $pagespeed_libs"
- CORE_INCS="$CORE_INCS $pagespeed_include"
- NGX_ADDON_SRCS="$PS_NGX_SRCS"
- if [ "$position_aux" = "true" ] ; then
- HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
- else
- HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES | sed "s/$next/$next $ngx_addon_name/")
- fi
-
- # Make the etag header filter run immediately before range header filter.
- HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
- sed "s/ngx_http_range_header_filter_module/ngx_http_range_header_filter_module ngx_pagespeed_etag_filter/")
-fi
-
-echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
-
-# Test whether the compiler is compatible
-ngx_feature="psol-compiler-compat"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs=""
-ngx_feature_path=""
-ngx_feature_libs="-lstdc++"
-ngx_feature_test="
-
-#if defined(__clang__) && defined(__GLIBCXX__)
- // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning
- // for a list of various values of __GLIBCXX__. Note that they're not monotonic
- // with respect to version numbers.
- #if __GLIBCXX__ == 20120322 || __GLIBCXX__ == 20120614
- #error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
- #endif
-#endif
-
-#if !defined(__clang__) && defined(__GNUC__)
- #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
- #error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
- #endif
-#endif
-
-#if defined(__clang__)
- #if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
- #error \"Please use gcc >= 4.8 or clang >= 3.3\"
- #endif
-#endif
-"
-
-. "$ngx_addon_dir/cpp_feature"
-
-if [ $ngx_found = no ]; then
- cat << END
-$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
-See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
-Look in $PWD/$NGX_AUTOCONF_ERR for more details.
-END
- exit 1
-fi
-
-have=NGX_PAGESPEED . auto/have
-
-CC_TEST_FLAGS="$CC_OLD_TEST_FLAGS"
diff --git a/lib/ngx_pagespeed/config.make b/lib/ngx_pagespeed/config.make
deleted file mode 100644
index d958c47..0000000
--- a/lib/ngx_pagespeed/config.make
+++ /dev/null
@@ -1,7 +0,0 @@
-# Since nginx build system doesn't normally do C++, there is no CXXFLAGS for us
-# to touch, and compilers are understandably unhappy with --std=c++11 on C
-# files. Hence, we hack the makefile to add it for just our sources.
-for ps_src_file in $PS_NGX_SRCS; do
- ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
- echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
-done
diff --git a/lib/ngx_pagespeed/cpp_feature b/lib/ngx_pagespeed/cpp_feature
deleted file mode 100644
index 4eb3985..0000000
--- a/lib/ngx_pagespeed/cpp_feature
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright (C) Igor Sysoev
-# Copyright (C) Nginx, Inc.
-# 2012-10-01 Modified from auto/feature by jefftk to support c++ test files.
-
-echo $ngx_n "checking for $ngx_feature ...$ngx_c"
-
-cat << END >> $NGX_AUTOCONF_ERR
-
-----------------------------------------
-checking for $ngx_feature
-
-END
-
-ngx_found=no
-
-if test -n "$ngx_feature_name"; then
- ngx_have_feature=`echo $ngx_feature_name \
- | tr abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ`
-fi
-
-if test -n "$ngx_feature_path"; then
- for ngx_temp in $ngx_feature_path; do
- ngx_feature_inc_path="$ngx_feature_inc_path -I $ngx_temp"
- done
-fi
-
-cat << END > $NGX_AUTOTEST.cc
-
-#include
-$NGX_INCLUDE_UNISTD_H
-$ngx_feature_incs
-
-int main() {
- $ngx_feature_test;
- return 0;
-}
-
-END
-
-
-ngx_test="$CC $CC_TEST_FLAGS $CC_AUX_FLAGS $ngx_feature_inc_path \
- -o $NGX_AUTOTEST $NGX_AUTOTEST.cc $NGX_TEST_LD_OPT $ngx_feature_libs"
-
-ngx_feature_inc_path=
-
-eval "/bin/sh -c \"$ngx_test\" >> $NGX_AUTOCONF_ERR 2>&1"
-
-
-if [ -x $NGX_AUTOTEST ]; then
-
- case "$ngx_feature_run" in
-
- yes)
- # /bin/sh is used to intercept "Killed" or "Abort trap" messages
- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
- echo " found"
- ngx_found=yes
-
- if test -n "$ngx_feature_name"; then
- have=$ngx_have_feature . auto/have
- fi
-
- else
- echo " found but is not working"
- fi
- ;;
-
- value)
- # /bin/sh is used to intercept "Killed" or "Abort trap" messages
- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
- echo " found"
- ngx_found=yes
-
- cat << END >> $NGX_AUTO_CONFIG_H
-
-#ifndef $ngx_feature_name
-#define $ngx_feature_name `$NGX_AUTOTEST`
-#endif
-
-END
- else
- echo " found but is not working"
- fi
- ;;
-
- bug)
- # /bin/sh is used to intercept "Killed" or "Abort trap" messages
- if /bin/sh -c $NGX_AUTOTEST >> $NGX_AUTOCONF_ERR 2>&1; then
- echo " not found"
-
- else
- echo " found"
- ngx_found=yes
-
- if test -n "$ngx_feature_name"; then
- have=$ngx_have_feature . auto/have
- fi
- fi
- ;;
-
- *)
- echo " found"
- ngx_found=yes
-
- if test -n "$ngx_feature_name"; then
- have=$ngx_have_feature . auto/have
- fi
- ;;
-
- esac
-
-else
- echo " not found"
-
- echo "----------" >> $NGX_AUTOCONF_ERR
- cat $NGX_AUTOTEST.cc >> $NGX_AUTOCONF_ERR
- echo "----------" >> $NGX_AUTOCONF_ERR
- echo $ngx_test >> $NGX_AUTOCONF_ERR
- echo "----------" >> $NGX_AUTOCONF_ERR
-fi
-
-rm $NGX_AUTOTEST*
diff --git a/lib/ngx_pagespeed/scripts/build_ngx_pagespeed.sh b/lib/ngx_pagespeed/scripts/build_ngx_pagespeed.sh
deleted file mode 100755
index 1b2c998..0000000
--- a/lib/ngx_pagespeed/scripts/build_ngx_pagespeed.sh
+++ /dev/null
@@ -1,814 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-function usage() {
- echo "
-Usage: build_ngx_pagespeed.sh [options]
-
- Installs ngx_pagespeed and its dependencies. Can optionally build and install
- nginx as well. Can be run either as:
-
- bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options]
-
- Or:
-
- git clone git@github.com:pagespeed/ngx_pagespeed.git
- cd ngx_pagespeed/
- git checkout
- scripts/build_ngx_pagespeed.sh [options]
-
-Options:
- -v, --ngx-pagespeed-version
- What version of ngx_pagespeed to build. Valid options include:
- * latest-beta
- * latest-stable
- * a version number, such as 1.11.33.4
-
- If you don't specify a version, defaults to latest-stable unless --devel
- is specified, in which case it defaults to master.
-
- This option doesn't make sense if we're running within an existing
- ngx_pagespeed checkout.
-
- -n, --nginx-version
- What version of nginx to build. If not set, this script only prepares the
- ngx_pagespeed module, and expects you to handle including it when you
- build nginx.
-
- If you pass in 'latest' then this script scrapes the nginx download page
- and attempts to determine the latest version automatically.
-
- -m, --dynamic-module
- Build ngx_pagespeed as a dynamic module.
-
- -b, --builddir
- Where to build. Defaults to \$HOME.
-
- -p, --no-deps-check
- By default, this script checks for the packages it depends on and tries to
- install them. If you have installed dependencies from source or are on a
- non-deb non-rpm system, this won't work. In that case, install the
- dependencies yourself and pass --no-deps-check.
-
- -s, --psol-from-source
- Build PSOL from source instead of downloading a pre-built binary module.
-
- -l, --devel
- Sets up a development environment in ngx_pagespeed/nginx, building with
- testing-only dependencies. Includes --psol-from-source, conflicts with
- --nginx-version. Uses a 'git clone' checkout for ngx_pagespeed and nginx
- instead of downloading a tarball.
-
- -t, --build-type
- When building PSOL from source, what to tell it for BUILD_TYPE. Defaults
- to 'Release' unless --devel is set in which case it defaults to 'Debug'.
-
- -y, --assume-yes
- Assume the answer to all prompts is 'yes, please continue'. Intended for
- automated usage, such as buildbots.
-
- -a, --additional-nginx-configure-arguments
- When running ./configure for nginx, you may want to specify additional
- arguments, such as --with-http_ssl_module. By default this script will
- pause and prompt you for them, but this option lets you pass them in. For
- example, you might do:
- -a '--with-http_ssl_module --with-cc-opt=\"-I /usr/local/include\"'
-
- -d, --dryrun
- Don't make any changes to the system, just print what changes you
- would have made.
-
- -h, --help
- Print this message and exit."
-}
-
-RED=31
-GREEN=32
-YELLOW=33
-function begin_color() {
- color="$1"
- echo -e -n "\e[${color}m"
-}
-function end_color() {
- echo -e -n "\e[0m"
-}
-
-function echo_color() {
- color="$1"
- shift
- begin_color "$color"
- echo "$@"
- end_color
-}
-
-function error() {
- local error_message="$@"
- echo_color "$RED" -n "Error: " >&2
- echo "$@" >&2
-}
-
-# Prints an error message and exits with an error code.
-function fail() {
- error "$@"
-
- # Normally I'd use $0 in "usage" here, but since most people will be running
- # this via curl, that wouldn't actually give something useful.
- echo >&2
- echo "For usage information, run this script with --help" >&2
- exit 1
-}
-
-
-function status() {
- echo_color "$GREEN" "$@"
-}
-
-# Intended to be called as:
-# bash <(curl dl.google.com/.../build_ngx_pagespeed.sh)
-
-# If we set -e or -u then users of this script will see it silently exit on
-# failure. Instead we need to check the exit status of each command manually.
-# The run function handles exit-status checking for system-changing commands.
-# Additionally, this allows us to easily have a dryrun mode where we don't
-# actually make any changes.
-INITIAL_ENV=$(printenv | sort)
-function run() {
- if "$DRYRUN"; then
- echo_color "$YELLOW" -n "would run"
- echo " $@"
- env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort))
- if [ -n "$env_differences" ]; then
- echo " with the following additional environment variables:"
- echo "$env_differences" | sed 's/^/ /'
- fi
- else
- if ! "$@"; then
- error "Failure running '$@', exiting."
- exit 1
- fi
- fi
-}
-
-function redhat_is_installed() {
- local package_name="$1"
- rpm -qa $package_name | grep -q .
-}
-
-function debian_is_installed() {
- local package_name="$1"
- dpkg -l $package_name | grep ^ii | grep -q .
-}
-
-function version_sort() {
- # We'd rather use sort -V, but that's not available on Centos 5. This works
- # for versions in the form A.B.C.D or shorter, which is enough for our use.
- sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g
-}
-
-# Compare two numeric versions in the form "A.B.C". Works with version numbers
-# having up to four components, since that's enough to handle both nginx (3) and
-# ngx_pagespeed (4).
-function version_older_than() {
- local test_version="$1"
- local compare_to="$2"
-
- local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1)
- test "$older_version" != "$compare_to"
-}
-
-function determine_latest_nginx_version() {
- # Scrape nginx's download page to try to find the most recent nginx version.
-
- nginx_download_url="https://nginx.org/en/download.html"
- function report_error() {
- fail "
-Couldn't automatically determine the latest nginx version: failed to $@
-$nginx_download_url"
- }
-
- nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \
- report_error "download"
-
- download_refs=$(echo "$nginx_download_page" | \
- grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \
- report_error "parse"
-
- versions_available=$(echo "$download_refs" | \
- sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \
- report_error "extract versions from"
-
- latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \
- report_error "determine latest version from"
-
- if version_older_than "$latest_version" "1.11.4"; then
- fail "
-Expected the latest version of nginx to be at least 1.11.4 but found
-$latest_version on $nginx_download_url"
- fi
-
- echo "$latest_version"
-}
-
-# Usage:
-# install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ...
-#
-# install_pkg_cmd is a command to install a dependency
-# is_pkg_installed_cmd is a command that returns true if the dependency is
-# already installed
-# each dependency is a package name
-function install_dependencies() {
- local install_pkg_cmd="$1"
- local is_pkg_installed_cmd="$2"
- shift 2
-
- local missing_dependencies=""
-
- for package_name in "$@"; do
- if ! $is_pkg_installed_cmd $package_name; then
- missing_dependencies+="$package_name "
- fi
- done
- if [ -n "$missing_dependencies" ]; then
- status "Detected that we're missing the following depencencies:"
- echo " $missing_dependencies"
- status "Installing them:"
- run sudo $install_pkg_cmd $missing_dependencies
- fi
-}
-
-function gcc_too_old() {
- # We need gcc >= 4.8
- local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}')
- if [ "$gcc_major_version" -lt 4 ]; then
- return 0 # too old
- elif [ "$gcc_major_version" -gt 4 ]; then
- return 1 # plenty new
- fi
- # It's gcc 4.x, check if x >= 8:
- local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}')
- test "$gcc_minor_version" -lt 8
-}
-
-function continue_or_exit() {
- if "$ASSUME_YES"; then
- return
- fi
-
- local prompt="$1"
- echo_color "$YELLOW" -n "$prompt"
- read -p " [Y/n] " yn
- if [[ "$yn" == N* || "$yn" == n* ]]; then
- echo "Cancelled."
- exit 0
- fi
-}
-
-# If a string is very simple we don't need to quote it. But we should quote
-# everything else to be safe.
-function needs_quoting() {
- echo "$@" | grep -q '[^a-zA-Z0-9./_=-]'
-}
-
-function escape_for_quotes() {
- echo "$@" | sed -e 's~\\~\\\\~g' -e "s~'~\\\\'~g"
-}
-
-function quote_arguments() {
- local argument_str=""
- for argument in "$@"; do
- if [ -n "$argument_str" ]; then
- argument_str+=" "
- fi
- if needs_quoting "$argument"; then
- argument="'$(escape_for_quotes "$argument")'"
- fi
- argument_str+="$argument"
- done
- echo "$argument_str"
-}
-
-function build_ngx_pagespeed() {
- getopt --test
- if [ "$?" != 4 ]; then
- # Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect
- # this to be hit in practice on systems that are actually able to run
- # PageSpeed.
- fail "Your version of getopt is too old. Exiting with no changes made."
- fi
-
- opts=$(getopt -o v:n:mb:pslt:ya:dh \
- --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \
- --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \
- --longoptions assume-yes,additional-nginx-configure-arguments:,dryrun,help \
- -n "$(basename "$0")" -- "$@")
- if [ $? != 0 ]; then
- usage
- exit 1
- fi
- eval set -- "$opts"
-
- NPS_VERSION="DEFAULT"
- NGINX_VERSION=""
- BUILDDIR="$HOME"
- DO_DEPS_CHECK=true
- PSOL_FROM_SOURCE=false
- DEVEL=false
- BUILD_TYPE=""
- ASSUME_YES=false
- DRYRUN=false
- DYNAMIC_MODULE=false
- while true; do
- case "$1" in
- -v | --ngx-pagespeed-version) shift
- NPS_VERSION="$1"
- shift
- ;;
- -n | --nginx-version) shift
- NGINX_VERSION="$1"
- shift
- ;;
- -m | --dynamic-module) shift
- DYNAMIC_MODULE=true
- ;;
- -b | --builddir) shift
- BUILDDIR="$1"
- shift
- ;;
- -p | --no-deps-check) shift
- DO_DEPS_CHECK=false
- ;;
- -s | --psol-from-source) shift
- PSOL_FROM_SOURCE=true
- ;;
- -l | --devel) shift
- DEVEL=true
- ;;
- -t | --build-type) shift
- BUILD_TYPE="$1"
- shift
- ;;
- -y | --assume-yes) shift
- ASSUME_YES="true"
- ;;
- -a | --additional-nginx-configure-arguments) shift
- ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS="$1"
- shift
- ;;
- -d | --dryrun) shift
- DRYRUN="true"
- ;;
- -h | --help) shift
- usage
- exit 0
- ;;
- --) shift
- break
- ;;
- *)
- echo "Invalid argument: $1"
- usage
- exit 1
- ;;
- esac
- done
-
- USE_GIT_CHECKOUT="$DEVEL"
- ALREADY_CHECKED_OUT=false
- if [ -e PSOL_BINARY_URL ]; then
- status "Detected that we're running in an existing ngx_pagespeed checkout."
- USE_GIT_CHECKOUT=true
- ALREADY_CHECKED_OUT=true
- fi
-
- if "$ALREADY_CHECKED_OUT"; then
- if [ "$NPS_VERSION" != "DEFAULT" ]; then
- fail \
-"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout."
- fi
- elif [ "$NPS_VERSION" = "DEFAULT" ]; then
- if "$DEVEL"; then
- NPS_VERSION="master"
- else
- NPS_VERSION="latest-stable"
- fi
- fi
-
- if [ ! -d "$BUILDDIR" ]; then
- fail "Told to build in $BUILDDIR, but that directory doesn't exist."
- fi
-
- BUILD_NGINX=false
- if [ -n "$NGINX_VERSION" ]; then
- BUILD_NGINX=true
- fi
-
- if "$DEVEL"; then
- PSOL_FROM_SOURCE=true
- BUILD_NGINX=true
- if [ -n "$NGINX_VERSION" ]; then
- fail \
-"The --devel argument conflicts with --nginx. In devel mode we use the version of nginx that's included as a submodule."
- fi
- if "$DYNAMIC_MODULE"; then
- fail "Can't currently build a dynamic module in --devel mode."
- fi
- fi
-
- if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then
- if "$DEVEL"; then
- BUILD_TYPE="Debug"
- else
- BUILD_TYPE="Release"
- fi
- elif [ -n "$BUILD_TYPE" ]; then
- fail "Setting --build-type requires --psol-from-source or --devel."
- fi
-
- if [ "$NGINX_VERSION" = "latest" ]; then
- # When this function fails it prints the debugging information needed first
- # to stderr.
- NGINX_VERSION=$(determine_latest_nginx_version) || exit 1
- fi
-
- if "$DYNAMIC_MODULE"; then
- # Check that ngx_pagespeed and nginx are recent enough to support dynamic
- # modules. Unfortunately NPS_VERSION might be a tag, in which case we don't
- # know. If it's not a numeric version number, then assume it's recent
- # enough and if it's not they'll get an ugly compilation error later.
- # Luckily 1.10.33.5 was a while ago now.
- #
- # I'd like to use =~ here, but they changed syntax between v3 and v4 (quotes
- # moved from mandatory to optional to prohibited).
- if [[ "${NPS_VERSION#*[^0-9.]}" = "$NPS_VERSION" ]] &&
- version_older_than "$NPS_VERSION" "1.10.33.5"; then
- fail "
-You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but
-ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5."
- fi
-
- if [ ! -z "NGINX_VERSION" ]; then
- if version_older_than "$NGINX_VERSION" "1.9.13"; then
- fail "
-You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't
-add support for dynamic modules in a way compatible with ngx_pagespeed until
-1.9.13."
- fi
- fi
- fi
-
- if "$DRYRUN"; then
- TEMPDIR="/tmp/output-of-mktemp"
- else
- TEMPDIR=$(mktemp -d)
- function cleanup_tempdir {
- rm -rf "$TEMPDIR"
- }
- trap cleanup_tempdir EXIT
- fi
-
- extra_flags=()
- # Now make sure our dependencies are installed.
- if "$DO_DEPS_CHECK"; then
- if [ -f /etc/debian_version ]; then
- status "Detected debian-based distro."
-
- install_dependencies "apt-get install" debian_is_installed \
- build-essential zlib1g-dev libpcre3 libpcre3-dev unzip uuid-dev
-
- if gcc_too_old; then
- if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
- status "Detected that gcc is older than 4.8. Installing gcc-mozilla"
- status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't"
- status "affect your global gcc installation."
- run sudo apt-get install gcc-mozilla
- fi
-
- extra_flags=("--with-cc=/usr/lib/gcc-mozilla/bin/gcc" \
- "--with-ld-opt=-static-libstdc++")
- fi
-
- elif [ -f /etc/redhat-release ]; then
- status "Detected redhat-based distro."
-
- install_dependencies "yum install" redhat_is_installed \
- gcc-c++ pcre-devel zlib-devel make unzip wget libuuid-devel
- if gcc_too_old; then
- if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
- redhat_major_version=$(
- cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1)
- if [ "$redhat_major_version" == 5 ]; then
- slc_version=5
- elif [ "$redhat_major_version" == 6 ]; then
- slc_version=6
- else
- fail "
-Unexpected major version $redhat_major_version in /etc/redhat-release:
-$(cat /etc/redhat-release) Expected 5 or 6."
- fi
-
- status "Detected that gcc is older than 4.8. Scientific Linux"
- status "provides a gcc package that installs gcc-4.8 into /opt/ and"
- status "doesn't affect your global gcc installation."
- slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/"
- slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern"
- slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern"
- run sudo wget "$slc_key" -O "$slc_key_out"
- run sudo rpm --import "$slc_key_out"
-
- repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo"
- if [ -e "$repo_fname" ]; then
- fail "Expected $repo_fname not to exist; aborting."
- fi
-
- repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/"
- repo_url+="/docs/repository/cern/devtoolset/"
- repo_url+="slc${slc_version}-devtoolset.repo"
- run sudo wget -O "$repo_fname" "$repo_url"
- run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils
- fi
- extra_flags=("--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc")
- fi
- else
- fail "
-This doesn't appear to be a deb-based distro or an rpm-based one. Not going to
-be able to install dependencies. Please install dependencies manually and rerun
-with --no-deps-check."
- fi
- status "Operating system dependencies are all set."
- else
- status "Not checking whether operating system dependencies are installed."
- fi
-
- function delete_if_already_exists() {
- if "$DRYRUN"; then return; fi
-
- local directory="$1"
- if [ -d "$directory" ]; then
- if [ ${#directory} -lt 8 ]; then
- fail "
-Not deleting $directory; name is suspiciously short. Something is wrong."
- fi
-
- continue_or_exit "OK to delete $directory?"
- run rm -rf "$directory"
- fi
- }
-
- # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo,
- # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION
- if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then
- # We've been given a numeric version number. This has an associated tag
- # in the form vVERSION-beta.
- tag_name="v${NPS_VERSION}-beta"
- nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta"
- else
- # We've been given a tag name, like latest-beta. Download that directly.
- tag_name="$NPS_VERSION"
- nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}"
- fi
-
- install_dir="this-only-makes-sense-in-devel-mode"
- if "$USE_GIT_CHECKOUT"; then
- # We're either doing a --devel build, or someone is running us from an
- # existing git checkout.
- nps_module_dir="$PWD"
- install_dir="$nps_module_dir"
- if "$ALREADY_CHECKED_OUT"; then
- run cd "$nps_module_dir"
- else
- status "Checking out ngx_pagespeed..."
- run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \
- "$nps_module_dir"
- run cd "$nps_module_dir"
- run git checkout "$tag_name"
- fi
- submodules_dir="$nps_module_dir/testing-dependencies"
- if "$DEVEL"; then
- status "Downloading dependencies..."
- run git submodule update --init --recursive
- if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then
- status "Switching submodules over to git protocol."
- # This lets us push to github by public key.
- for config in $(find .git/ -name config) ; do
- run sed -i s~https://github.com/~git@github.com:~ $config ;
- done
- fi
- fi
- else
- nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive"
- nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip"
- status "Downloading ngx_pagespeed..."
- run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded"
- nps_module_dir="$BUILDDIR/$nps_downloaded_fname"
- delete_if_already_exists "$nps_module_dir"
- status "Extracting ngx_pagespeed..."
- run unzip -q "$nps_downloaded" -d "$BUILDDIR"
- run cd "$nps_module_dir"
- fi
-
- MOD_PAGESPEED_DIR=""
- PSOL_BINARY=""
- if "$PSOL_FROM_SOURCE"; then
- MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed"
- git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR"
- run pushd "$MOD_PAGESPEED_DIR"
-
- if "$DEVEL"; then
- if [ ! -d "$HOME/apache2" ]; then
- run install/build_development_apache.sh 2.2 prefork
- fi
- cd devel
- run make apache_debug_psol
- PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
- else
- if "$DO_DEPS_CHECK"; then
- skip_deps_arg=""
- else
- skip_deps_arg="--skip_deps"
- fi
-
- run install/build_psol.sh --skip_tests --skip_packaging "$skip_deps_arg"
- PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
- fi
- run popd
- else
- # Now we need to figure out what precompiled version of PSOL to build
- # ngx_pagespeed against.
- if "$DRYRUN"; then
- psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz"
- elif [ -e PSOL_BINARY_URL ]; then
- # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us
- # where to look.
- psol_url="$(scripts/format_binary_url.sh PSOL_BINARY_URL)"
- if [[ "$psol_url" != https://* ]]; then
- fail "Got bad psol binary location information: $psol_url"
- fi
- else
- # For past releases we have to grep it from the config file. The url has
- # always looked like this, and the config file has contained it since
- # before we started tagging our ngx_pagespeed releases.
- psol_url="$(grep -o \
- "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)"
- if [ -z "$psol_url" ]; then
- fail "Couldn't find PSOL url in $PWD/config"
- fi
- fi
-
- status "Downloading PSOL binary..."
- run wget "$psol_url"
-
- status "Extracting PSOL..."
- run tar -xzf $(basename "$psol_url") # extracts to psol/
- fi
-
- if "$DYNAMIC_MODULE"; then
- add_module="--add-dynamic-module=$nps_module_dir"
- else
- add_module="--add-module=$nps_module_dir"
- fi
- configure_args=("$add_module" "${extra_flags[@]}")
-
- if "$DEVEL"; then
- configure_args=("${configure_args[@]}"
- "--prefix=$install_dir/nginx"
- "--add-module=$submodules_dir/ngx_cache_purge"
- "--add-module=$submodules_dir/ngx_devel_kit"
- "--add-module=$submodules_dir/set-misc-nginx-module"
- "--add-module=$submodules_dir/headers-more-nginx-module"
- "--with-ipv6"
- "--with-http_v2_module")
- if [ "$BUILD_TYPE" = "Debug" ]; then
- configure_args=("${configure_args[@]}" "--with-debug")
- fi
- fi
-
- echo
- if ! "$BUILD_NGINX"; then
- # Just prepare the module for them to install.
- status "ngx_pagespeed is ready to be built against nginx."
- echo "When running ./configure:"
- if "$PSOL_FROM_SOURCE"; then
- echo " Set the following environment variables:"
- echo " MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR"
- echo " PSOL_BINARY=$PSOL_BINARY"
- fi
- echo " Give ./configure the following arguments:"
- echo " $(quote_arguments "${configure_args[@]}")"
- echo
- if [ ${#extra_flags[@]} -eq 0 ]; then
- echo "If this is for integration with an already-built nginx, make sure"
- echo "to include any other arguments you originally passed to"
- echo "./configure. You can see these with 'nginx -V'."
- else
- echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")"
- echo "on this platform, if you want to integrate ngx_pagespeed with an"
- echo "already-built nginx you're going to need to rebuild your nginx with"
- echo "those flags set."
- fi
- else
- if "$DEVEL"; then
- # Use the nginx we loaded as a submodule
- nginx_dir="$submodules_dir/nginx"
- configure_location="auto"
- else
- # Download and build the specified nginx version.
- nginx_leaf="nginx-${NGINX_VERSION}.tar.gz"
- nginx_fname="$TEMPDIR/$nginx_leaf"
- status "Downloading nginx..."
- run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname"
- nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/"
- delete_if_already_exists "$nginx_dir"
- status "Extracting nginx..."
- run tar -xzf "$nginx_fname" --directory "$BUILDDIR"
- configure_location="."
- fi
- run cd "$nginx_dir"
-
- configure=("$configure_location/configure" "${configure_args[@]}")
- additional_configure_args=""
- if [ -z "${ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS+x}" ]; then
- if ! "$ASSUME_YES"; then
- echo "About to build nginx. Do you have any additional ./configure"
- echo "arguments you would like to set? For example, if you would like"
- echo "to build nginx with https support give --with-http_ssl_module"
- echo "If you don't have any, just press enter."
- read -p "> " additional_configure_args
- fi
- else
- additional_configure_args="$ADDITIONAL_NGINX_CONFIGURE_ARGUMENTS"
- fi
-
- if [ -n "$additional_configure_args" ]; then
- # Split additional_configure_args respecting any internal quotation.
- # Otherwise things like --with-cc-opt='-foo -bar' won't work.
- eval additional_configure_args=("$additional_configure_args")
- configure=("${configure[@]}" "${additional_configure_args[@]}")
- fi
-
- echo "About to configure nginx with:"
- echo " $(quote_arguments "${configure[@]}")"
- continue_or_exit "Does this look right?"
- MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
- PSOL_BINARY="$PSOL_BINARY" \
- run "${configure[@]}"
-
- if ! "$DEVEL"; then
- continue_or_exit "Build nginx?"
- fi
- run make
-
- if "$DEVEL"; then
- run make install
-
- status "Nginx installed with ngx_pagespeed, and set up for development."
- echo "To run tests:"
- echo " cd $nps_module_dir"
- echo " test/run_tests.sh"
- echo
- echo "To rebuild after changes:"
- echo " scripts/rebuild.sh"
- else
- continue_or_exit "Install nginx?"
- run sudo make install
-
- echo
- if "$DYNAMIC_MODULE"; then
- echo "Nginx installed with ngx_pagespeed support available as a"
- echo "loadable module."
- echo
- echo "To load the ngx_pagespeed module, you'll need to add:"
- echo " load_module \"modules/ngx_pagespeed.so\";"
- echo "at the top of your main nginx configuration file."
- else
- echo "Nginx installed with ngx_pagespeed support compiled-in."
- fi
- echo
- echo "If this is a new installation you probably need an init script to"
- echo "manage starting and stopping the nginx service. See:"
- echo " http://wiki.nginx.org/InitScripts"
- echo
- echo "You'll also need to configure ngx_pagespeed if you haven't yet:"
- echo " https://developers.google.com/speed/pagespeed/module/configuration"
- fi
- fi
- if "$DRYRUN"; then
- echo_color "$YELLOW" "[this was a dry run; your system is unchanged]"
- fi
-}
-
-# Start running things from a call at the end so if this script is executed
-# after a partial download it doesn't do anything.
-build_ngx_pagespeed "$@"
diff --git a/lib/ngx_pagespeed/scripts/format_binary_url.sh b/lib/ngx_pagespeed/scripts/format_binary_url.sh
deleted file mode 100755
index 870100e..0000000
--- a/lib/ngx_pagespeed/scripts/format_binary_url.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-set -u
-
-if [ $# -ne 1 ]; then
- echo "Usage: $(basename $0) " >&2
- exit 1
-fi
-
-url_file=$1
-
-if [ ! -e "$url_file" ]; then
- echo "Url file '$url_file' missing!" >&2
-fi
-
-# The size names must match install/build_psol.sh in mod_pagespeed
-if [ "$(uname -m)" = x86_64 ]; then
- bit_size_name=x64
-else
- bit_size_name=ia32
-fi
-
-sed -e 's/$BIT_SIZE_NAME\b/'$bit_size_name'/g' $url_file
diff --git a/lib/ngx_pagespeed/scripts/pagespeed_libraries_generator.sh b/lib/ngx_pagespeed/scripts/pagespeed_libraries_generator.sh
deleted file mode 100755
index bd9ac36..0000000
--- a/lib/ngx_pagespeed/scripts/pagespeed_libraries_generator.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2013 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
-# supporting the canonicalize_javascript_libraries filter.
-# Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532
-#
-# Usage:
-# scripts/pagespeed_libraries_generator.sh > pagespeed_libraries.conf
-#
-# Then have nginx include that configuration file and enable the filter
-# canonicalize_javascript_libraries.
-#
-# Author: vid@zippykid.com (Vid Luther)
-# jefftk@google.com (Jeff Kaufman)
-
-URL="https://github.com/pagespeed/mod_pagespeed/raw/master/"
-URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
-curl -L -s -S "$URL" \
- | grep ModPagespeedLibrary \
- | while read library size hash url ; do
- echo " pagespeed Library $size $hash $url;"
-done
diff --git a/lib/ngx_pagespeed/scripts/rebuild.sh b/lib/ngx_pagespeed/scripts/rebuild.sh
deleted file mode 100755
index deda07d..0000000
--- a/lib/ngx_pagespeed/scripts/rebuild.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2016 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Usage: scripts/rebuild.sh
-#
-# After building with "scripts/build_ngx_pagespeed.sh --devel", if you make
-# changes to ngx_pagespeed you'll need to rebuild it. The underlying commands
-# aren't complicated, but it's faster to work if it's automated.
-
-set -e # exit script if any command returns an error
-set -u # exit the script if any variable is uninitialized
-
-this_dir="$(dirname $0)"
-cd "$this_dir/.."
-nps_dir="$PWD"
-
-cd "$nps_dir/testing-dependencies/mod_pagespeed/devel"
-make apache_debug_psol
-
-cd "$nps_dir/testing-dependencies/nginx/"
-make
-make install
diff --git a/lib/ngx_pagespeed/src/log_message_handler.cc b/lib/ngx_pagespeed/src/log_message_handler.cc
deleted file mode 100644
index 060339d..0000000
--- a/lib/ngx_pagespeed/src/log_message_handler.cc
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jmarantz@google.com (Joshua Marantz)
-// Author: sligocki@google.com (Shawn Ligocki)
-// Author: jefftk@google.com (Jeff Kaufman)
-
-// TODO(jefftk): share more of this code with apache's log_message_handler
-
-#include "log_message_handler.h"
-
-#include
-
-#include
-#include
-
-#include "base/debug/debugger.h"
-#include "base/debug/stack_trace.h"
-#include "base/logging.h"
-#include "net/instaweb/public/version.h"
-#include "pagespeed/kernel/base/string_util.h"
-
-// Make sure we don't attempt to use LOG macros here, since doing so
-// would cause us to go into an infinite log loop.
-#undef LOG
-#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
-
-namespace {
-
-ngx_log_t* ngx_log = NULL;
-
-ngx_uint_t GetNgxLogLevel(int severity) {
- switch (severity) {
- case logging::LOG_INFO:
- return NGX_LOG_INFO;
- case logging::LOG_WARNING:
- return NGX_LOG_WARN;
- case logging::LOG_ERROR:
- return NGX_LOG_ERR;
- case logging::LOG_ERROR_REPORT:
- case logging::LOG_FATAL:
- return NGX_LOG_ALERT;
- default: // For VLOG(s)
- return NGX_LOG_DEBUG;
- }
-}
-
-bool LogMessageHandler(int severity, const char* file, int line,
- size_t message_start, const GoogleString& str) {
- ngx_uint_t this_log_level = GetNgxLogLevel(severity);
-
- GoogleString message = str;
- if (severity == logging::LOG_FATAL) {
- if (base::debug::BeingDebugged()) {
- base::debug::BreakDebugger();
- } else {
- base::debug::StackTrace trace;
- std::ostringstream stream;
- trace.OutputToStream(&stream);
- message.append(stream.str());
- }
- }
-
- // Trim the newline off the end of the message string.
- size_t last_msg_character_index = message.length() - 1;
- if (message[last_msg_character_index] == '\n') {
- message.resize(last_msg_character_index);
- }
-
- ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
- net_instaweb::kModPagespeedVersion,
- message.c_str());
-
- if (severity == logging::LOG_FATAL) {
- // Crash the process to generate a dump.
- base::debug::BreakDebugger();
- }
-
- return true;
-}
-
-} // namespace
-
-
-namespace net_instaweb {
-
-namespace log_message_handler {
-
-
-void Install(ngx_log_t* log_in) {
- ngx_log = log_in;
- logging::SetLogMessageHandler(&LogMessageHandler);
-
- // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
- // level is DEBUG.
- if (ngx_log->log_level >= NGX_LOG_DEBUG) {
- logging::SetMinLogLevel(-2);
- }
-}
-
-} // namespace log_message_handler
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/log_message_handler.h b/lib/ngx_pagespeed/src/log_message_handler.h
deleted file mode 100644
index 05f7277..0000000
--- a/lib/ngx_pagespeed/src/log_message_handler.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2010 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jmarantz@google.com (Joshua Marantz)
-// Author: sligocki@google.com (Shawn Ligocki)
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#ifndef LOG_MESSAGE_HANDLER_H_
-#define LOG_MESSAGE_HANDLER_H_
-
-extern "C" {
- #include
- #include
- #include
- #include
-}
-
-namespace net_instaweb {
-
-namespace log_message_handler {
-
-// Install a log message handler that routes LOG() messages to the
-// server error log. Should be called once at startup. If server blocks define
-// their own logging files you would expect that LOG() messages would be routed
-// appropriately, but because logging is all done with global variables this
-// isn't possible.
-void Install(ngx_log_t* log_in);
-
-} // namespace log_message_handler
-
-} // namespace net_instaweb
-
-#endif // LOG_MESSAGE_HANDLER_H_
diff --git a/lib/ngx_pagespeed/src/ngx_base_fetch.cc b/lib/ngx_pagespeed/src/ngx_base_fetch.cc
deleted file mode 100644
index 8c6c31d..0000000
--- a/lib/ngx_pagespeed/src/ngx_base_fetch.cc
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * Author: jefftk@google.com (Jeff Kaufman)
- */
-
-#include "ngx_pagespeed.h" // Must come first, see comments in CollectHeaders.
-
-#include //for usleep
-
-#include "ngx_base_fetch.h"
-#include "ngx_event_connection.h"
-#include "ngx_list_iterator.h"
-
-#include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/rewriter/public/rewrite_stats.h"
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/http/response_headers.h"
-
-namespace net_instaweb {
-
-const char kHeadersComplete = 'H';
-const char kFlush = 'F';
-const char kDone = 'D';
-
-NgxEventConnection* NgxBaseFetch::event_connection = NULL;
-int NgxBaseFetch::active_base_fetches = 0;
-
-NgxBaseFetch::NgxBaseFetch(StringPiece url,
- ngx_http_request_t* r,
- NgxServerContext* server_context,
- const RequestContextPtr& request_ctx,
- PreserveCachingHeaders preserve_caching_headers,
- NgxBaseFetchType base_fetch_type,
- const RewriteOptions* options)
- : AsyncFetch(request_ctx),
- url_(url.data(), url.size()),
- request_(r),
- server_context_(server_context),
- options_(options),
- need_flush_(false),
- done_called_(false),
- last_buf_sent_(false),
- references_(2),
- base_fetch_type_(base_fetch_type),
- preserve_caching_headers_(preserve_caching_headers),
- detached_(false),
- suppress_(false) {
- if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
- __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
-}
-
-NgxBaseFetch::~NgxBaseFetch() {
- pthread_mutex_destroy(&mutex_);
- __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
-}
-
-bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
- CHECK(event_connection == NULL) << "event connection already set";
- event_connection = new NgxEventConnection(ReadCallback);
- return event_connection->Init(cycle);
-}
-
-void NgxBaseFetch::Terminate() {
- if (event_connection != NULL) {
- GoogleMessageHandler handler;
- PosixTimer timer;
- int64 timeout_us = Timer::kSecondUs * 30;
- int64 end_us = timer.NowUs() + timeout_us;
- static unsigned int sleep_microseconds = 100;
-
- handler.Message(
- kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
- NgxBaseFetch::active_base_fetches);
-
- // Try to continue processing and get the active base fetch count to 0
- // untill the timeout expires.
- // TODO(oschaaf): This needs more work.
- while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
- event_connection->Drain();
- usleep(sleep_microseconds);
- }
-
- if (NgxBaseFetch::active_base_fetches != 0) {
- handler.Message(
- kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
- NgxBaseFetch::active_base_fetches);
- }
-
- // Close down the named pipe.
- event_connection->Shutdown();
- delete event_connection;
- event_connection = NULL;
- }
-}
-
-const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
- switch(type) {
- case kPageSpeedResource:
- return "ps resource";
- case kHtmlTransform:
- return "html transform";
- case kAdminPage:
- return "admin page";
- case kIproLookup:
- return "ipro lookup";
- case kPageSpeedProxy:
- return "pagespeed proxy";
- }
- CHECK(false);
- return "can't get here";
-}
-
-// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
-// MessageHandler and use that.
-void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
- NgxBaseFetch* base_fetch = reinterpret_cast(data.sender);
- ngx_http_request_t* r = base_fetch->request();
- bool detached = base_fetch->detached();
-#if (NGX_DEBUG) // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
- const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
-#endif
- int refcount = base_fetch->DecrementRefCount();
-
-#if (NGX_DEBUG)
- ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
- "pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
- data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
-#endif
-
- // If we ended up destructing the base fetch, or the request context is
- // detached, skip this event.
- if (refcount == 0 || detached) {
- return;
- }
-
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- // If our request context was zeroed, skip this event.
- // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
- if (ctx == NULL) {
- // Should not happen normally, when it does this message will cause our
- // system tests to fail.
- ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
- "pagespeed [%p] skipping event: request context gone", r);
- return;
- }
-
- // Normally we expect the sender to equal the active NgxBaseFetch instance.
- DCHECK(data.sender == ctx->base_fetch);
-
- // If someone changed our request context or NgxBaseFetch, skip processing.
- if (data.sender != ctx->base_fetch) {
- ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
- "pagespeed [%p] skipping event: event originating from disassociated"
- " NgxBaseFetch instance.", r);
- return;
- }
-
- int rc;
- bool run_posted = true;
- // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
- // we must enter a different flow. Also see ps_in_place_check_header_filter().
- if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
- && r->connection->error) {
- ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
- "pagespeed [%p] request already finalized %d", r, r->count);
- rc = NGX_ERROR;
- run_posted = false;
- } else {
- rc = ps_base_fetch::ps_base_fetch_handler(r);
- }
-
-#if (NGX_DEBUG)
- ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
- "pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
- r, rc, data.type);
-#endif
-
- ngx_connection_t* c = r->connection;
- ngx_http_finalize_request(r, rc);
-
- if (run_posted) {
- // See http://forum.nginx.org/read.php?2,253006,253061
- ngx_http_run_posted_requests(c);
- }
-}
-
-void NgxBaseFetch::Lock() {
- pthread_mutex_lock(&mutex_);
-}
-
-void NgxBaseFetch::Unlock() {
- pthread_mutex_unlock(&mutex_);
-}
-
-bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
- MessageHandler* handler) {
- Lock();
- buffer_.append(sp.data(), sp.size());
- Unlock();
- return true;
-}
-
-// should only be called in nginx thread
-ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
- CHECK(!(done_called_ && last_buf_sent_))
- << "CopyBufferToNginx() was called after the last buffer was sent";
-
- // there is no buffer to send
- if (!done_called_ && buffer_.empty()) {
- *link_ptr = NULL;
- return NGX_AGAIN;
- }
-
- int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
- done_called_ /* send_last_buf */,
- need_flush_);
- need_flush_ = false;
-
- if (rc != NGX_OK) {
- return rc;
- }
-
- // Done with buffer contents now.
- buffer_.clear();
-
- if (done_called_) {
- last_buf_sent_ = true;
- return NGX_OK;
- }
-
- return NGX_AGAIN;
-}
-
-// There may also be a race condition if this is called between the last Write()
-// and Done() such that we're sending an empty buffer with last_buf set, which I
-// think nginx will reject.
-ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
- ngx_int_t rc;
- Lock();
- rc = CopyBufferToNginx(link_ptr);
- Unlock();
- return rc;
-}
-
-ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
- // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
- // If a standard header is accidentally included before the nginx header,
- // on a 32-bit system off_t will be 4 bytes and we don't assign all the
- // bits of content_length_n. Sanity check that did not happen.
- // This could use static_assert, but this file is not built with --std=c++11.
- bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
-
- const ResponseHeaders* pagespeed_headers = response_headers();
-
- if (content_length_known()) {
- headers_out->content_length = NULL;
- headers_out->content_length_n = content_length();
- }
-
- return copy_response_headers_to_ngx(request_, *pagespeed_headers,
- preserve_caching_headers_);
-}
-
-void NgxBaseFetch::RequestCollection(char type) {
- if (suppress_) {
- return;
- }
-
- // We must optimistically increment the refcount, and decrement it
- // when we conclude we failed. If we only increment on a successfull write,
- // there's a small chance that between writing and adding to the refcount
- // both pagespeed and nginx will release their refcount -- destructing
- // this NgxBaseFetch instance.
- IncrementRefCount();
- if (!event_connection->WriteEvent(type, this)) {
- DecrementRefCount();
- }
-}
-
-void NgxBaseFetch::HandleHeadersComplete() {
- int status_code = response_headers()->status_code();
- bool status_ok = (status_code != 0) && (status_code < 400);
-
- if ((base_fetch_type_ != kIproLookup) || status_ok) {
- // If this is a 404 response we need to count it in the stats.
- if (response_headers()->status_code() == HttpStatus::kNotFound) {
- server_context_->rewrite_stats()->resource_404_count()->Add(1);
- }
- }
-
- RequestCollection(kHeadersComplete); // Headers available.
-
- // For the IPRO lookup, supress notification of the nginx side here.
- // If we send both the headerscomplete event and the one from done, nasty
- // stuff will happen if we loose the race with with the nginx side destructing
- // this base fetch instance.
- if (base_fetch_type_ == kIproLookup && !status_ok) {
- suppress_ = true;
- }
-}
-
-bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
- Lock();
- need_flush_ = true;
- Unlock();
- RequestCollection(kFlush); // A new part of the response body is available
- return true;
-}
-
-int NgxBaseFetch::DecrementRefCount() {
- return DecrefAndDeleteIfUnreferenced();
-}
-
-int NgxBaseFetch::IncrementRefCount() {
- return __sync_add_and_fetch(&references_, 1);
-}
-
-int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
- // Creates a full memory barrier.
- int r = __sync_add_and_fetch(&references_, -1);
- if (r == 0) {
- delete this;
- }
- return r;
-}
-
-void NgxBaseFetch::HandleDone(bool success) {
- // TODO(jefftk): it's possible that instead of locking here we can just modify
- // CopyBufferToNginx to only read done_called_ once.
- CHECK(!done_called_) << "Done already called!";
- Lock();
- done_called_ = true;
- Unlock();
- RequestCollection(kDone);
- DecrefAndDeleteIfUnreferenced();
-}
-
-bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
- return OptionsAwareHTTPCacheCallback::IsCacheValid(
- url_, *options_, request_context(), headers);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_base_fetch.h b/lib/ngx_pagespeed/src/ngx_base_fetch.h
deleted file mode 100644
index 13db33e..0000000
--- a/lib/ngx_pagespeed/src/ngx_base_fetch.h
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-//
-// Collects output from pagespeed and buffers it until nginx asks for it.
-// Notifies nginx via NgxEventConnection to call ReadCallback() when
-// the headers are computed, when a flush should be performed, and when done.
-//
-// - nginx creates a base fetch and passes it to a new proxy fetch.
-// - The proxy fetch manages rewriting and thread complexity, and through
-// several chained steps passes rewritten html to HandleWrite().
-// - Written data is buffered.
-// - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
-// PSOL, events are written to NgxEventConnection which will end up being
-// handled by ReadCallback() on nginx's thread.
-// When applicable, request processing will be continued via a call to
-// ps_base_fetch_handler().
-// - ps_base_fetch_handler() will pull the header and body bytes from PSOL
-// via CollectAccumulatedWrites() and write those to the module's output.
-//
-// This class is referred to in three places: the proxy fetch, nginx's request,
-// and pending events written to the associated NgxEventConnection. It must stay
-// alive until the proxy fetch and nginx request are finished, and no more
-// events are pending.
-// - The proxy fetch will call Done() to indicate this.
-// - nginx will call Detach() when the associated request is handled
-// completely (e.g. the request context is about to be destroyed).
-// - ReadCallback() will call DecrementRefCount() on instances associated to
-// events it handles.
-//
-// When the last reference is dropped, this class will delete itself.
-//
-// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
-// with ApacheFetch, using a common base class.
-
-#ifndef NGX_BASE_FETCH_H_
-#define NGX_BASE_FETCH_H_
-
-extern "C" {
-#include
-}
-
-#include
-
-#include "ngx_pagespeed.h"
-
-#include "ngx_event_connection.h"
-#include "ngx_server_context.h"
-
-#include "net/instaweb/http/public/async_fetch.h"
-#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/headers.h"
-
-namespace net_instaweb {
-
-enum NgxBaseFetchType {
- kIproLookup,
- kHtmlTransform,
- kPageSpeedResource,
- kAdminPage,
- kPageSpeedProxy
-};
-
-class NgxBaseFetch : public AsyncFetch {
- public:
- NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
- NgxServerContext* server_context,
- const RequestContextPtr& request_ctx,
- PreserveCachingHeaders preserve_caching_headers,
- NgxBaseFetchType base_fetch_type,
- const RewriteOptions* options);
- virtual ~NgxBaseFetch();
-
- // Statically initializes event_connection, require for PSOL and nginx to
- // communicate.
- static bool Initialize(ngx_cycle_t* cycle);
-
- // Attempts to finish up request processing queued up in the named pipe and
- // PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
- // is attempted.
- // Statically terminates and NULLS event_connection.
- static void Terminate();
-
- static void ReadCallback(const ps_event_data& data);
-
- // Puts a chain in link_ptr if we have any output data buffered. Returns
- // NGX_OK on success, NGX_ERROR on errors. If there's no data to send, sends
- // data only if Done() has been called. Indicates the end of output by
- // setting last_buf on the last buffer in the chain.
- //
- // Sets link_ptr to a chain of as many buffers are needed for the output.
- //
- // Called by nginx in response to seeing a byte on the pipe.
- ngx_int_t CollectAccumulatedWrites(ngx_chain_t** link_ptr);
-
- // Copies response headers into headers_out.
- //
- // Called by nginx before calling CollectAccumulatedWrites() for the first
- // time for resource fetches. Not called at all for proxy fetches.
- ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
-
- // Called by nginx to decrement the refcount.
- int DecrementRefCount();
-
- // Called by pagespeed to increment the refcount.
- int IncrementRefCount();
-
- // Detach() is called when the nginx side releases this base fetch. It
- // sets detached_ to true and decrements the refcount. We need to know
- // this to be able to handle events which nginx request context has been
- // released while the event was in-flight.
- void Detach() { detached_ = true; DecrementRefCount(); }
-
- bool detached() { return detached_; }
-
- ngx_http_request_t* request() { return request_; }
- NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
-
- bool IsCachedResultValid(const ResponseHeaders& headers) override;
-
- private:
- virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
- virtual bool HandleFlush(MessageHandler* handler);
- virtual void HandleHeadersComplete();
- virtual void HandleDone(bool success);
-
- // Indicate to nginx that we would like it to call
- // CollectAccumulatedWrites().
- void RequestCollection(char type);
-
- // Lock must be acquired first.
- // Returns:
- // NGX_ERROR: failure
- // NGX_AGAIN: still has buffer to send, need to checkout link_ptr
- // NGX_OK: done, HandleDone has been called
- // Allocates an nginx buffer, copies our buffer_ contents into it, clears
- // buffer_.
- ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
-
- void Lock();
- void Unlock();
-
- // Called by Done() and Release(). Decrements our reference count, and if
- // it's zero we delete ourself.
- int DecrefAndDeleteIfUnreferenced();
-
- static NgxEventConnection* event_connection;
-
- // Live count of NgxBaseFetch instances that are currently in use.
- static int active_base_fetches;
-
- GoogleString url_;
- ngx_http_request_t* request_;
- GoogleString buffer_;
- NgxServerContext* server_context_;
- const RewriteOptions* options_;
- bool need_flush_;
- bool done_called_;
- bool last_buf_sent_;
- // How many active references there are to this fetch. Starts at two,
- // decremented once when Done() is called and once when Detach() is called.
- // Incremented for each event written by pagespeed for this NgxBaseFetch, and
- // decremented on the nginx side for each event read for it.
- int references_;
- pthread_mutex_t mutex_;
- NgxBaseFetchType base_fetch_type_;
- PreserveCachingHeaders preserve_caching_headers_;
- // Set to true just before the nginx side releases its reference
- bool detached_;
- bool suppress_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_BASE_FETCH_H_
diff --git a/lib/ngx_pagespeed/src/ngx_caching_headers.cc b/lib/ngx_pagespeed/src/ngx_caching_headers.cc
deleted file mode 100644
index 164f825..0000000
--- a/lib/ngx_pagespeed/src/ngx_caching_headers.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#include "ngx_caching_headers.h"
-#include "ngx_list_iterator.h"
-
-#include "ngx_pagespeed.h"
-
-namespace net_instaweb {
-
-bool NgxCachingHeaders::Lookup(const StringPiece& key,
- StringPieceVector* values) {
- ngx_table_elt_t* header;
- NgxListIterator it(&(request_->headers_out.headers.part));
- while ((header = it.Next()) != NULL) {
- if (header->hash != 0 && key == str_to_string_piece(header->key)) {
- // This will be called multiple times if there are multiple headers with
- // this name. Each time it will append to values.
- SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
- true /* omit empty strings */);
- }
- }
-
- if (values->size() == 0) {
- // No header found with this name.
- return false;
- }
-
- for (int i = 0, n = values->size(); i < n; ++i) {
- TrimWhitespace(&((*values)[i]));
- }
-
- return true;
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_caching_headers.h b/lib/ngx_pagespeed/src/ngx_caching_headers.h
deleted file mode 100644
index 44cebb9..0000000
--- a/lib/ngx_pagespeed/src/ngx_caching_headers.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#ifndef NGX_CACHING_HEADERS_H_
-#define NGX_CACHING_HEADERS_H_
-
-extern "C" {
- #include
-}
-
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/http/caching_headers.h"
-
-namespace net_instaweb {
-
-// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
-// Needed so ps_header_filter can call GenerateDisabledCacheControl().
-class NgxCachingHeaders : public CachingHeaders {
- public:
- explicit NgxCachingHeaders(ngx_http_request_t* request)
- : CachingHeaders(request->headers_out.status),
- request_(request) {
- }
-
- virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
-
- virtual bool IsLikelyStaticResourceType() const {
- DCHECK(false); // not called in our use-case.
- return false;
- }
-
- virtual bool IsCacheableResourceStatusCode() const {
- DCHECK(false); // not called in our use-case.
- return false;
- }
-
- private:
- ngx_http_request_t* request_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_CACHING_HEADERS_H_
diff --git a/lib/ngx_pagespeed/src/ngx_event_connection.cc b/lib/ngx_pagespeed/src/ngx_event_connection.cc
deleted file mode 100644
index 6ab88f9..0000000
--- a/lib/ngx_pagespeed/src/ngx_event_connection.cc
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: oschaaf@we-amp.com (Otto van der Schaaf)
-
-extern "C" {
-
-#include
-
-}
-
-#include "ngx_event_connection.h"
-
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/message_handler.h"
-
-namespace net_instaweb {
-
- NgxEventConnection::NgxEventConnection(callbackPtr callback)
- : event_handler_(callback) {
-}
-
-bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
- int file_descriptors[2];
-
- if (pipe(file_descriptors) != 0) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
- return false;
- }
- if (ngx_nonblocking(file_descriptors[0]) == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
- ngx_nonblocking_n "pagespeed: pipe[0] failed");
- } else if (ngx_nonblocking(file_descriptors[1]) == -1) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
- ngx_nonblocking_n "pagespeed: pipe[1] failed");
- } else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
- ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
- "pagespeed: failed to create connection.");
- } else {
- pipe_read_fd_ = file_descriptors[0];
- pipe_write_fd_ = file_descriptors[1];
- // Attempt to bump the pipe capacity, because running out of buffer space
- // can potentially lead up to writes spinning on EAGAIN.
- // See https://github.com/pagespeed/ngx_pagespeed/issues/1380
- // TODO(oschaaf): Consider implementing a queueing mechanism for retrying
- // failed writes.
-#ifdef F_SETPIPE_SZ
- fcntl(pipe_write_fd_, F_SETPIPE_SZ, 200*1024 /* minimal amount of bytes */);
-#endif
- return true;
- }
- close(file_descriptors[0]);
- close(file_descriptors[1]);
- return false;
-}
-
-bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
- ngx_fd_t pipe_fd) {
- // pipe_fd (the read side of the pipe will end up as c->fd on the
- // underlying ngx_connection_t that gets created here)
- ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
- &NgxEventConnection::ReadEventHandler);
- return rc == NGX_OK;
-}
-
-void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
- ngx_connection_t* c = static_cast(ev->data);
- ngx_int_t result = ngx_handle_read_event(ev, 0);
- if (result != NGX_OK) {
- CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
- }
-
- if (ev->timedout) {
- ev->timedout = 0;
- return;
- }
-
- if (!NgxEventConnection::ReadAndNotify(c->fd)) {
- // This was copied from ngx_channel_handler(): for epoll, we need to call
- // ngx_del_conn(). Sadly, no documentation as to why.
- if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
- ngx_del_conn(c, 0);
- }
- ngx_close_connection(c);
- ngx_del_event(ev, NGX_READ_EVENT, 0);
- }
-}
-
-// Deserialize ps_event_data's from the pipe as they become available.
-// Subsequently do some bookkeeping, cleanup, and error checking to keep
-// the mess out of ps_base_fetch_handler.
-bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
- while (true) {
- // We read only one ps_event_data at a time for now:
- // We can end up recursing all the way and end up calling ourselves here.
- // If that happens in the middle of looping over multiple ps_event_data's we
- // have obtained with read(), the results from the next read() will make us
- // process events out of order. Which can give headaches.
- // Alternatively, we could maintain a queue to make sure we process in
- // sequence
- ps_event_data data;
- ngx_int_t size = read(fd, static_cast(&data), sizeof(data));
-
- if (size == -1) {
- if (errno == EINTR) {
- continue;
- // TODO(oschaaf): should we worry about spinning here?
- } else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
- return true;
- }
- }
-
- if (size <= 0) {
- return false;
- }
-
- data.connection->event_handler_(data);
- return true;
- }
-}
-
-bool NgxEventConnection::WriteEvent(void* sender) {
- return WriteEvent('X' /* Anything char is fine */, sender);
-}
-
-bool NgxEventConnection::WriteEvent(char type, void* sender) {
- ssize_t size = 0;
- ps_event_data data;
-
- ngx_memzero(&data, sizeof(data));
- data.type = type;
- data.sender = sender;
- data.connection = this;
-
- while (true) {
- size = write(pipe_write_fd_,
- static_cast(&data), sizeof(data));
- if (size == sizeof(data)) {
- return true;
- } else if (size == -1) {
- // TODO(oschaaf): should we worry about spinning here?
- if (ngx_errno == EINTR || ngx_errno == EAGAIN
- || ngx_errno == EWOULDBLOCK) {
- continue;
- } else {
- return false;
- }
- } else {
- CHECK(false) << "pagespeed: unexpected return value from write(): "
- << size;
- }
- }
- CHECK(false) << "Should not get here";
- return false;
-}
-
-// Reads and processes what is available in the pipe.
-void NgxEventConnection::Drain() {
- NgxEventConnection::ReadAndNotify(pipe_read_fd_);
-}
-
-void NgxEventConnection::Shutdown() {
- close(pipe_write_fd_);
- close(pipe_read_fd_);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_event_connection.h b/lib/ngx_pagespeed/src/ngx_event_connection.h
deleted file mode 100644
index 3ce46a1..0000000
--- a/lib/ngx_pagespeed/src/ngx_event_connection.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: oschaaf@we-amp.com (Otto van der Schaaf)
-//
-// NgxEventConnection implements a means to send events from other threads to
-// nginx's event loop, and is implemented by a named pipe under the hood.
-// A single instance is used by NgxBaseFetch, and one instance is created per
-// NgxUrlAsyncFetcher when native fetching is on.
-
-#ifndef NGX_EVENT_CONNECTION_H_
-#define NGX_EVENT_CONNECTION_H_
-
-extern "C" {
-#include
-}
-
-#include
-
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/headers.h"
-
-namespace net_instaweb {
-
-class NgxEventConnection;
-
-// Represents a single event that can be written to or read from the pipe.
-// Technically, sender is the only data we need to send. type and connection are
-// included to provide a means to trace the events along with some more
-// info.
-typedef struct {
- char type;
- void* sender;
- NgxEventConnection* connection;
-} ps_event_data;
-
-// Handler signature for receiving events
-typedef void (*callbackPtr)(const ps_event_data&);
-
-// Abstracts a connection to nginx through which events can be written.
-class NgxEventConnection {
- public:
- explicit NgxEventConnection(callbackPtr handler);
-
- // Creates the file descriptors and ngx_connection_t required for event
- // messaging between pagespeed and nginx.
- bool Init(ngx_cycle_t* cycle);
- // Shuts down the underlying file descriptors and connection created in Init()
- void Shutdown();
- // Constructs a ps_event_data and writes it to the underlying named pipe.
- bool WriteEvent(char type, void* sender);
- // Convenience overload for clients that have a single event type.
- bool WriteEvent(void* sender);
- // Reads and processes what is available in the named pipe's buffer.
- void Drain();
- private:
- static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
- static void ReadEventHandler(ngx_event_t* e);
- static bool ReadAndNotify(ngx_fd_t fd);
-
- callbackPtr event_handler_;
- // We own these file descriptors
- ngx_fd_t pipe_write_fd_;
- ngx_fd_t pipe_read_fd_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_EVENT_CONNECTION_H_
diff --git a/lib/ngx_pagespeed/src/ngx_fetch.cc b/lib/ngx_pagespeed/src/ngx_fetch.cc
deleted file mode 100644
index 90e5bd3..0000000
--- a/lib/ngx_pagespeed/src/ngx_fetch.cc
+++ /dev/null
@@ -1,951 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: x.dinic@gmail.com (Junmin Xiong)
-//
-// - The fetch is started by the main thread.
-// - Resolver event was hooked when a NgxFetch starting. It could
-// lookup the IP of the domain asynchronously from the DNS server.
-// - When NgxFetchResolveDone is called, It will create the request and the
-// connection. Add the write and read event to the epoll structure.
-// - The read handler parses the response. Add the response to the buffer at
-// last.
-
-// TODO(oschaaf): Currently the first applicable connection is picked from the
-// pool when re-using connections. Perhaps it would be worth it to pick the one
-// that was active the longest time ago to keep a larger pool available.
-// TODO(oschaaf): style: reindent namespace according to google C++ style guide
-// TODO(oschaaf): Retry mechanism for failures on a re-used k-a connection.
-// Currently we don't think it's going to be an issue, see the comments at
-// https://github.com/pagespeed/ngx_pagespeed/pull/781.
-
-extern "C" {
-#include
-}
-
-#include "ngx_fetch.h"
-
-#include "base/logging.h"
-
-#include
-#include
-#include
-#include
-
-#include "net/instaweb/http/public/async_fetch.h"
-#include "net/instaweb/http/public/inflating_fetch.h"
-#include "net/instaweb/public/version.h"
-#include "net/instaweb/public/global_constants.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/condvar.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/pool_element.h"
-#include "pagespeed/kernel/base/scoped_ptr.h"
-#include "pagespeed/kernel/base/statistics.h"
-#include "pagespeed/kernel/base/string_writer.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/thread_system.h"
-#include "pagespeed/kernel/base/timer.h"
-#include "pagespeed/kernel/base/writer.h"
-#include "pagespeed/kernel/http/request_headers.h"
-#include "pagespeed/kernel/http/response_headers.h"
-#include "pagespeed/kernel/http/response_headers_parser.h"
-
-namespace net_instaweb {
-
-NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
-PthreadMutex NgxConnection::connection_pool_mutex;
-// Default keepalive 60s.
-const int64 NgxConnection::keepalive_timeout_ms = 60000;
-const GoogleString NgxConnection::ka_header =
- StrCat("keep-alive ",
- Integer64ToString(NgxConnection::keepalive_timeout_ms));
-
-NgxConnection::NgxConnection(MessageHandler* handler,
- int max_keepalive_requests) {
- c_ = NULL;
- max_keepalive_requests_ = max_keepalive_requests;
- handler_ = handler;
- // max_keepalive_requests specifies the number of http requests that are
- // allowed to be performed over a single connection. So, a
- // max_keepalive_requests of 1 effectively disables keepalive.
- keepalive_ = max_keepalive_requests_ > 1;
-}
-
-NgxConnection::~NgxConnection() {
- CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
-}
-
-void NgxConnection::Terminate() {
- for (NgxConnectionPool::iterator p = connection_pool.begin();
- p != connection_pool.end(); ++p) {
- NgxConnection* nc = *p;
- ngx_close_connection(nc->c_);
- nc->c_ = NULL;
- delete nc;
- }
- connection_pool.Clear();
-}
-
-NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
- MessageHandler* handler,
- int max_keepalive_requests) {
- NgxConnection* nc;
- {
- ScopedMutex lock(&NgxConnection::connection_pool_mutex);
-
- for (NgxConnectionPool::iterator p = connection_pool.begin();
- p != connection_pool.end(); ++p) {
- nc = *p;
-
- if (ngx_memn2cmp(static_cast(nc->sockaddr_),
- reinterpret_cast(pc->sockaddr),
- nc->socklen_, pc->socklen) == 0) {
- CHECK(nc->c_->idle) << "Pool should only contain idle connections!";
-
- nc->c_->idle = 0;
- nc->c_->log = pc->log;
- nc->c_->read->log = pc->log;
- nc->c_->write->log = pc->log;
- if (nc->c_->pool != NULL) {
- nc->c_->pool->log = pc->log;
- }
-
- if (nc->c_->read->timer_set) {
- ngx_del_timer(nc->c_->read);
- }
- connection_pool.Remove(nc);
-
- ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
- "NgxFetch: re-using connection %p (pool size: %l)",
- nc, connection_pool.size());
- return nc;
- }
- }
- }
-
- int rc = ngx_event_connect_peer(pc);
- if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
- return NULL;
- }
-
- // NgxConnection deletes itself if NgxConnection::Close()
- nc = new NgxConnection(handler, max_keepalive_requests);
- nc->SetSock(reinterpret_cast(pc->sockaddr), pc->socklen);
- nc->c_ = pc->connection;
- return nc;
-}
-
-void NgxConnection::Close() {
- bool removed_from_pool = false;
-
- {
- ScopedMutex lock(&NgxConnection::connection_pool_mutex);
- for (NgxConnectionPool::iterator p = connection_pool.begin();
- p != connection_pool.end(); ++p) {
- if (*p == this) {
- // When we get here, that means that the connection either has timed
- // out or has been closed remotely.
- connection_pool.Remove(this);
- ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
- "NgxFetch: removed connection %p (pool size: %l)",
- this, connection_pool.size());
- removed_from_pool = true;
- break;
- }
- }
- }
-
- max_keepalive_requests_--;
-
- if (c_->read->timer_set) {
- ngx_del_timer(c_->read);
- }
-
- if (c_->write->timer_set) {
- ngx_del_timer(c_->write);
- }
-
- if (!keepalive_ || max_keepalive_requests_ <= 0 || removed_from_pool) {
- ngx_close_connection(c_);
- c_ = NULL;
- delete this;
- return;
- }
-
- ngx_add_timer(c_->read, static_cast(
- NgxConnection::keepalive_timeout_ms));
-
- c_->data = this;
- c_->read->handler = NgxConnection::IdleReadHandler;
- c_->write->handler = NgxConnection::IdleWriteHandler;
- c_->idle = 1;
-
- // This connection should not be associated with current fetch.
- c_->log = ngx_cycle->log;
- c_->read->log = ngx_cycle->log;
- c_->write->log = ngx_cycle->log;
- if (c_->pool != NULL) {
- c_->pool->log = ngx_cycle->log;
- }
-
- // Allow this connection to be re-used, by adding it to the connection pool.
- {
- ScopedMutex lock(&NgxConnection::connection_pool_mutex);
- connection_pool.Add(this);
- ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
- "NgxFetch: Added connection %p (pool size: %l - "
- " max_keepalive_requests_ %d)",
- this, connection_pool.size(), max_keepalive_requests_);
- }
-}
-
-void NgxConnection::IdleWriteHandler(ngx_event_t* ev) {
- ngx_connection_t* c = static_cast(ev->data);
- u_char buf[1];
- int n = c->recv(c, buf, 1);
- if (c->write->timedout) {
- DCHECK(false) << "NgxFetch: write timeout not expected." << n;
- }
- if (n == NGX_AGAIN) {
- return;
- }
-}
-
-void NgxConnection::IdleReadHandler(ngx_event_t* ev) {
- ngx_connection_t* c = static_cast(ev->data);
- NgxConnection* nc = static_cast(c->data);
-
- if (c->read->timedout) {
- nc->set_keepalive(false);
- nc->Close();
- return;
- }
-
- char buf[1];
- int n;
-
- // not a timeout event, we should check connection
- n = recv(c->fd, buf, 1, MSG_PEEK);
- if (n == -1 && ngx_socket_errno == NGX_EAGAIN) {
- if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
- nc->set_keepalive(false);
- nc->Close();
- return;
- }
-
- return;
- }
-
- nc->set_keepalive(false);
- nc->Close();
-}
-
-NgxFetch::NgxFetch(const GoogleString& url,
- AsyncFetch* async_fetch,
- MessageHandler* message_handler,
- ngx_log_t* log)
- : str_url_(url),
- fetcher_(NULL),
- async_fetch_(async_fetch),
- parser_(async_fetch->response_headers()),
- message_handler_(message_handler),
- bytes_received_(0),
- fetch_start_ms_(0),
- fetch_end_ms_(0),
- done_(false),
- content_length_(-1),
- content_length_known_(false),
- resolver_ctx_(NULL) {
- ngx_memzero(&url_, sizeof(url_));
- log_ = log;
- pool_ = NULL;
- timeout_event_ = NULL;
- connection_ = NULL;
-}
-
-NgxFetch::~NgxFetch() {
- if (timeout_event_ != NULL && timeout_event_->timer_set) {
- ngx_del_timer(timeout_event_);
- }
- if (connection_ != NULL) {
- connection_->Close();
- connection_ = NULL;
- }
- if (pool_ != NULL) {
- ngx_destroy_pool(pool_);
- pool_ = NULL;
- }
-}
-
-// This function is called by NgxUrlAsyncFetcher::StartFetch.
-bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
- fetcher_ = fetcher;
- bool ok = Init();
- if (ok) {
- ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
- this);
- } // else Init() will have emitted a reason
- return ok;
-}
-
-// Create the pool, parse the url, add the timeout event and
-// hook the DNS resolver if needed. Else we connect directly.
-// When this returns false, our caller (NgxUrlAsyncFetcher::StartFetch)
-// will call fetch->CallbackDone()
-bool NgxFetch::Init() {
- pool_ = ngx_create_pool(12288, log_);
- if (pool_ == NULL) {
- message_handler_->Message(kError, "NgxFetch: ngx_create_pool failed");
- return false;
- }
-
- if (!ParseUrl()) {
- message_handler_->Message(kError,
- "NgxFetch: ParseUrl() failed for [%s]:%s",
- str_url_.c_str(), url_.err);
- return false;
- }
-
- timeout_event_ = static_cast(
- ngx_pcalloc(pool_, sizeof(ngx_event_t)));
- if (timeout_event_ == NULL) {
- message_handler_->Message(kError,
- "NgxFetch: ngx_pcalloc failed for timeout");
- return false;
- }
-
- timeout_event_->data = this;
- timeout_event_->handler = NgxFetch::TimeoutHandler;
- timeout_event_->log = log_;
-
- ngx_add_timer(timeout_event_, fetcher_->fetch_timeout_);
- r_ = static_cast(
- ngx_pcalloc(pool_, sizeof(ngx_http_request_t)));
-
- if (r_ == NULL) {
- message_handler_->Message(kError,
- "NgxFetch: ngx_pcalloc failed for timer");
- return false;
- }
- status_ = static_cast(
- ngx_pcalloc(pool_, sizeof(ngx_http_status_t)));
-
- if (status_ == NULL) {
- message_handler_->Message(kError,
- "NgxFetch: ngx_pcalloc failed for status");
- return false;
- }
-
- // The host is either a domain name or an IP address. First check
- // if it's a valid IP address and only if that fails fall back to
- // using the DNS resolver.
-
- // Maybe we have a Proxy.
- ngx_url_t* tmp_url = &url_;
- if (fetcher_->proxy_.url.len != 0) {
- tmp_url = &fetcher_->proxy_;
- }
-
- GoogleString s_ipaddress(reinterpret_cast(tmp_url->host.data),
- tmp_url->host.len);
- ngx_memzero(&sin_, sizeof(sin_));
- sin_.sin_family = AF_INET;
- sin_.sin_port = htons(tmp_url->port);
- sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
-
- if (sin_.sin_addr.s_addr == INADDR_NONE) {
- // inet_addr returned INADDR_NONE, which means the hostname
- // isn't a valid IP address. Check DNS.
- ngx_resolver_ctx_t temp;
- temp.name.data = tmp_url->host.data;
- temp.name.len = tmp_url->host.len;
- resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
- if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
- // TODO(oschaaf): this spams the log, but is useful in the fetcher's
- // current state
- message_handler_->Message(
- kError, "NgxFetch: Couldn't start resolving, "
- "is there a proper resolver configured in nginx.conf?");
- return false;
- } else {
- ngx_log_error(NGX_LOG_DEBUG, log_, 0,
- "NgxFetch %p: start resolve for: %s",
- this, s_ipaddress.c_str());
- }
-
- resolver_ctx_->data = this;
- resolver_ctx_->name.data = tmp_url->host.data;
- resolver_ctx_->name.len = tmp_url->host.len;
-
-#if (nginx_version < 1005008)
- resolver_ctx_->type = NGX_RESOLVE_A;
-#endif
-
- resolver_ctx_->handler = NgxFetch::ResolveDoneHandler;
- resolver_ctx_->timeout = fetcher_->resolver_timeout_;
-
- if (ngx_resolve_name(resolver_ctx_) != NGX_OK) {
- message_handler_->Message(kWarning,
- "NgxFetch: ngx_resolve_name failed");
- return false;
- }
- } else {
- if (InitRequest() != NGX_OK) {
- message_handler()->Message(kError, "NgxFetch: InitRequest failed");
- return false;
- }
- }
- return true;
-}
-
-const char* NgxFetch::str_url() {
- return str_url_.c_str();
-}
-
-// This function should be called only once. The only argument is sucess or
-// not.
-void NgxFetch::CallbackDone(bool success) {
- ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
- this, success ? "OK":"FAIL");
-
- if (async_fetch_ == NULL) {
- LOG(FATAL)
- << "BUG: NgxFetch callback called more than once on same fetch"
- << str_url_.c_str() << "(" << this << ").Please report this at"
- << "https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss";
- return;
- }
-
- release_resolver();
-
- if (timeout_event_ && timeout_event_->timer_set) {
- ngx_del_timer(timeout_event_);
- timeout_event_ = NULL;
- }
-
- if (connection_ != NULL) {
- // Connection will be re-used only on responses that specify
- // 'Connection: keep-alive' in their headers.
- bool keepalive = false;
-
- if (success) {
- ConstStringStarVector v;
- if (async_fetch_->response_headers()->Lookup(
- StringPiece(HttpAttributes::kConnection), &v)) {
- for (size_t i = 0; i < v.size(); i++) {
- if (*v[i] == "keep-alive") {
- keepalive = true;
- break;
- } else if (*v[i] == "close") {
- break;
- }
- }
- }
- ngx_log_error(NGX_LOG_DEBUG, log_, 0,
- "NgxFetch %p: connection %p attempt keep-alive: %s",
- this, connection_, keepalive ? "Yes":"No");
- }
-
- connection_->set_keepalive(keepalive);
- connection_->Close();
- connection_ = NULL;
- }
-
- if (fetcher_ != NULL) {
- if (fetcher_->track_original_content_length()
- && async_fetch_->response_headers()->Has(
- HttpAttributes::kXOriginalContentLength)) {
- async_fetch_->extra_response_headers()->SetOriginalContentLength(
- bytes_received_);
- }
- fetcher_->FetchComplete(this);
- }
- async_fetch_->Done(success);
- async_fetch_ = NULL;
-}
-
-size_t NgxFetch::bytes_received() {
- return bytes_received_;
-}
-void NgxFetch::bytes_received_add(int64 x) {
- bytes_received_ += x;
-}
-
-int64 NgxFetch::fetch_start_ms() {
- return fetch_start_ms_;
-}
-
-void NgxFetch::set_fetch_start_ms(int64 start_ms) {
- fetch_start_ms_ = start_ms;
-}
-
-int64 NgxFetch::fetch_end_ms() {
- return fetch_end_ms_;
-}
-
-void NgxFetch::set_fetch_end_ms(int64 end_ms) {
- fetch_end_ms_ = end_ms;
-}
-
-MessageHandler* NgxFetch::message_handler() {
- return message_handler_;
-}
-
-bool NgxFetch::ParseUrl() {
- url_.url.len = str_url_.length();
- url_.url.data = static_cast(ngx_palloc(pool_, url_.url.len));
- if (url_.url.data == NULL) {
- return false;
- }
- str_url_.copy(reinterpret_cast(url_.url.data), str_url_.length(), 0);
-
- return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
-}
-
-// Issue a request after the resolver is done
-void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
- NgxFetch* fetch = static_cast(resolver_ctx->data);
- NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
-
- if (resolver_ctx->state != NGX_OK) {
- if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
- ngx_del_timer(fetch->timeout_event());
- fetch->set_timeout_event(NULL);
- }
- fetch->message_handler()->Message(
- kWarning, "NgxFetch %p: failed to resolve host [%.*s]", fetch,
- static_cast(resolver_ctx->name.len), resolver_ctx->name.data);
- fetch->CallbackDone(false);
- return;
- }
-
- ngx_uint_t i;
- // Find the first ipv4 address. We don't support ipv6 yet.
- for (i = 0; i < resolver_ctx->naddrs; i++) {
- // Old versions of nginx and tengine have a different definition of addrs,
- // work around to make sure we are using the right type (ngx_addr_t*).
- ngx_addr_t* ngx_addrs = reinterpret_cast(resolver_ctx->addrs);
- if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
- if (reinterpret_cast(ngx_addrs[i].sockaddr)
- ->sin_family == AF_INET) {
- break;
- }
- } else {
- // We're using an old version that uses in_addr_t* for addrs.
- break;
- }
- }
-
- // If no suitable ipv4 address was found, we fail.
- if (i == resolver_ctx->naddrs) {
- if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
- ngx_del_timer(fetch->timeout_event());
- fetch->set_timeout_event(NULL);
- }
- fetch->message_handler()->Message(
- kWarning, "NgxFetch %p: no suitable address for host [%.*s]", fetch,
- static_cast(resolver_ctx->name.len), resolver_ctx->name.data);
- fetch->CallbackDone(false);
- }
-
- ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
-
-#if (nginx_version < 1005008)
- fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
-#else
- struct sockaddr_in* sin;
-
- sin = reinterpret_cast(
- resolver_ctx->addrs[i].sockaddr);
-
- fetch->sin_.sin_family = sin->sin_family;
- fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
-#endif
-
- fetch->sin_.sin_family = AF_INET;
- fetch->sin_.sin_port = htons(fetch->url_.port);
-
- // Maybe we have Proxy
- if (0 != fetcher->proxy_.url.len) {
- fetch->sin_.sin_port = htons(fetcher->proxy_.port);
- }
-
- char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
-
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: Resolved host [%V] to [%s]", fetch,
- &resolver_ctx->name, ip_address);
-
- fetch->release_resolver();
-
- if (fetch->InitRequest() != NGX_OK) {
- fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
- fetch->CallbackDone(false);
- }
-}
-
-// Prepare the request data for this fetch, and hook the write event.
-int NgxFetch::InitRequest() {
- in_ = ngx_create_temp_buf(pool_, 4096);
- if (in_ == NULL) {
- return NGX_ERROR;
- }
-
- FixUserAgent();
-
- RequestHeaders* request_headers = async_fetch_->request_headers();
- ConstStringStarVector v;
- size_t size = 0;
- bool have_host = false;
- GoogleString port;
-
- response_handler = NgxFetch::HandleStatusLine;
- int rc = Connect();
- if (rc == NGX_AGAIN || rc == NGX_OK) {
- if (connection_->keepalive()) {
- request_headers->Add(HttpAttributes::kConnection,
- NgxConnection::ka_header);
- }
- const char* method = request_headers->method_string();
- size_t method_len = strlen(method);
-
- size = (method_len +
- 1 /* for the space */ +
- url_.uri.len +
- sizeof(" HTTP/1.0\r\n") - 1);
-
- for (int i = 0; i < request_headers->NumAttributes(); i++) {
- // if no explicit host header is given in the request headers,
- // we need to derive it from the url.
- if (StringCaseEqual(request_headers->Name(i), "Host")) {
- have_host = true;
- }
-
- // name: value\r\n
- size += request_headers->Name(i).length()
- + request_headers->Value(i).length() + 4; // 4 for ": \r\n"
- }
-
- if (!have_host) {
- port = StrCat(":", IntegerToString(url_.port));
- // for "Host: " + host + ":" + port + "\r\n"
- size += url_.host.len + 8 + port.size();
- }
-
- size += 2; // "\r\n";
- out_ = ngx_create_temp_buf(pool_, size);
-
- if (out_ == NULL) {
- return NGX_ERROR;
- }
-
- out_->last = ngx_cpymem(out_->last, method, method_len);
- out_->last = ngx_cpymem(out_->last, " ", 1);
- out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
- out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
-
- if (!have_host) {
- out_->last = ngx_cpymem(out_->last, "Host: ", 6);
- out_->last = ngx_cpymem(out_->last, url_.host.data, url_.host.len);
- out_->last = ngx_cpymem(out_->last, port.c_str(), port.size());
- out_->last = ngx_cpymem(out_->last, "\r\n", 2);
- }
-
- for (int i = 0; i < request_headers->NumAttributes(); i++) {
- const GoogleString& name = request_headers->Name(i);
- const GoogleString& value = request_headers->Value(i);
- out_->last = ngx_cpymem(out_->last, name.c_str(), name.length());
- *(out_->last++) = ':';
- *(out_->last++) = ' ';
- out_->last = ngx_cpymem(out_->last, value.c_str(), value.length());
- *(out_->last++) = CR;
- *(out_->last++) = LF;
- }
- *(out_->last++) = CR;
- *(out_->last++) = LF;
- if (rc == NGX_AGAIN) {
- return NGX_OK;
- }
- } else if (rc < NGX_OK) {
- return rc;
- }
- CHECK(rc == NGX_OK);
- NgxFetch::ConnectionWriteHandler(connection_->c_->write);
- return NGX_OK;
-}
-
-int NgxFetch::Connect() {
- ngx_peer_connection_t pc;
- ngx_memzero(&pc, sizeof(pc));
- pc.sockaddr = (struct sockaddr*)&sin_;
- pc.socklen = sizeof(struct sockaddr_in);
- pc.name = &url_.host;
-
- // get callback is dummy function, it just returns NGX_OK
- pc.get = ngx_event_get_peer;
- pc.log_error = NGX_ERROR_ERR;
- pc.log = fetcher_->log_;
- pc.rcvbuf = -1;
-
-
- connection_ = NgxConnection::Connect(&pc, message_handler(),
- fetcher_->max_keepalive_requests_);
- ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
- "NgxFetch %p Connect() connection %p for [%s]",
- this, connection_, str_url());
-
- if (connection_ == NULL) {
- return NGX_ERROR;
- }
-
- connection_->c_->write->handler = NgxFetch::ConnectionWriteHandler;
- connection_->c_->read->handler = NgxFetch::ConnectionReadHandler;
- connection_->c_->data = this;
-
- // Timer set in Init() is still in effect.
- return NGX_OK;
-}
-
-// When the fetch sends the request completely, it will hook the read event,
-// and prepare to parse the response. Timer set in Init() is still in effect.
-void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
- ngx_connection_t* c = static_cast(wev->data);
- NgxFetch* fetch = static_cast(c->data);
- ngx_buf_t* out = fetch->out_;
- bool ok = true;
- while (wev->ready && out->pos < out->last) {
- int n = c->send(c, out->pos, out->last - out->pos);
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: ConnectionWriteHandler "
- "send result %d", fetch, n);
-
- if (n >= 0) {
- out->pos += n;
- } else if (n == NGX_AGAIN) {
- break;
- } else {
- ok = false;
- break;
- }
- }
-
- if (ok) {
- if (out->pos == out->last) {
- ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
- } else {
- ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
- }
- }
-
- if (!ok) {
- fetch->message_handler()->Message(
- kWarning, "NgxFetch %p: failed to hook next event", fetch);
- c->error = 1;
- fetch->CallbackDone(false);
- }
-}
-
-// Timer set in Init() is still in effect.
-void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
- ngx_connection_t* c = static_cast(rev->data);
- NgxFetch* fetch = static_cast(c->data);
- bool ok = true;
-
- while(rev->ready) {
- int n = c->recv(
- c, fetch->in_->start, fetch->in_->end - fetch->in_->start);
-
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: ConnectionReadHandler "
- "recv result %d", fetch, n);
-
- if (n == NGX_AGAIN) {
- break;
- } else if (n == 0) {
- // If the content length was not known, we assume that we have read
- // all if we at least parsed the headers.
- // If we do know the content length, having a mismatch on the bytes read
- // will be interpreted as an error.
- ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
- || fetch->parser_.headers_complete();
- fetch->done_ = true;
- break;
- } else if (n > 0) {
- fetch->in_->pos = fetch->in_->start;
- fetch->in_->last = fetch->in_->start + n;
- ok = fetch->response_handler(c);
- if (fetch->done_ || !ok) {
- break;
- }
- }
- }
-
- if (!ok) {
- fetch->CallbackDone(false);
- } else if (fetch->done_) {
- fetch->CallbackDone(true);
- } else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
- fetch->CallbackDone(false);
- }
-}
-
-// Parse the status line: "HTTP/1.1 200 OK\r\n"
-bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
- NgxFetch* fetch = static_cast(c->data);
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: Handle status line", fetch);
-
- // This function only works after Nginx-1.1.4. Before nginx-1.1.4,
- // ngx_http_parse_status_line didn't save http_version.
- ngx_int_t n = ngx_http_parse_status_line(fetch->r_, fetch->in_,
- fetch->status_);
- if (n == NGX_ERROR) { // parse status line error
- fetch->message_handler()->Message(
- kWarning, "NgxFetch: failed to parse status line");
- return false;
- } else if (n == NGX_AGAIN) { // not completed
- return true;
- }
-
- ResponseHeaders* response_headers =
- fetch->async_fetch_->response_headers();
- response_headers->SetStatusAndReason(
- static_cast(fetch->get_status_code()));
- response_headers->set_major_version(fetch->get_major_version());
- response_headers->set_minor_version(fetch->get_minor_version());
-
- fetch->in_->pos += n;
- fetch->set_response_handler(NgxFetch::HandleHeader);
- if ((fetch->in_->last - fetch->in_->pos) > 0) {
- return fetch->response_handler(c);
- }
- return true;
-}
-
-// Parse the HTTP headers
-bool NgxFetch::HandleHeader(ngx_connection_t* c) {
- NgxFetch* fetch = static_cast(c->data);
- char* data = reinterpret_cast(fetch->in_->pos);
- size_t size = fetch->in_->last - fetch->in_->pos;
- size_t n = fetch->parser_.ParseChunk(StringPiece(data, size),
- fetch->message_handler_);
-
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: Handle headers", fetch);
-
- if (n > size) {
- return false;
- } else if (fetch->parser_.headers_complete()) {
- // TODO(oschaaf): We should also check if the request method was HEAD
- // - but I don't think PSOL uses that at this point.
- if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
- fetch->done_ = true;
- } else if (fetch->async_fetch_->response_headers()->FindContentLength(
- &fetch->content_length_)) {
- if (fetch->content_length_ < 0) {
- fetch->message_handler_->Message(
- kError, "Negative content-length in response header");
- return false;
- } else {
- fetch->content_length_known_ = true;
- if (fetch->content_length_ == 0) {
- fetch->done_ = true;
- }
- }
- }
-
- if (fetch->fetcher_->track_original_content_length()
- && fetch->content_length_known_) {
- fetch->async_fetch_->response_headers()->SetOriginalContentLength(
- fetch->content_length_);
- }
-
- fetch->in_->pos += n;
- if (!fetch->done_) {
- fetch->set_response_handler(NgxFetch::HandleBody);
- if ((fetch->in_->last - fetch->in_->pos) > 0) {
- return fetch->response_handler(c);
- }
- }
- } else {
- fetch->in_->pos += n;
- }
- return true;
-}
-
-// Read the response body
-bool NgxFetch::HandleBody(ngx_connection_t* c) {
- NgxFetch* fetch = static_cast(c->data);
- char* data = reinterpret_cast(fetch->in_->pos);
- size_t size = fetch->in_->last - fetch->in_->pos;
-
- fetch->bytes_received_add(size);
-
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: Handle body (%d bytes)", fetch, size);
-
- if ( fetch->async_fetch_->Write(StringPiece(data, size),
- fetch->message_handler()) ) {
- if (fetch->bytes_received_ == fetch->content_length_) {
- fetch->done_ = true;
- }
- fetch->in_->pos += size;
- } else {
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: async fetch write failure", fetch);
- return false;
- }
- return true;
-}
-
-void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
- NgxFetch* fetch = static_cast(tev->data);
- ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
- "NgxFetch %p: TimeoutHandler called", fetch);
- fetch->CallbackDone(false);
-}
-
-void NgxFetch::FixUserAgent() {
- GoogleString user_agent;
- ConstStringStarVector v;
- RequestHeaders* request_headers = async_fetch_->request_headers();
- if (request_headers->Lookup(HttpAttributes::kUserAgent, &v)) {
- for (size_t i = 0, n = v.size(); i < n; i++) {
- if (i != 0) {
- user_agent += " ";
- }
-
- if (v[i] != NULL) {
- user_agent += *(v[i]);
- }
- }
- request_headers->RemoveAll(HttpAttributes::kUserAgent);
- }
- if (user_agent.empty()) {
- user_agent += "NgxNativeFetcher";
- }
- GoogleString version = StrCat(
- " (", kModPagespeedSubrequestUserAgent,
- "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
- if (!StringPiece(user_agent).ends_with(version)) {
- user_agent += version;
- }
- request_headers->Add(HttpAttributes::kUserAgent, user_agent);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_fetch.h b/lib/ngx_pagespeed/src/ngx_fetch.h
deleted file mode 100644
index ae91e49..0000000
--- a/lib/ngx_pagespeed/src/ngx_fetch.h
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: x.dinic@gmail.com (Junmin Xiong)
-//
-// PageSpeed needs some way to talk to the internet and request resources. For
-// example, if it's optimizing www.example.com/index.html and it sees html with
-// 
and images.example.com is authorized
-// for rewriting in the config, then it needs to fetch cat.jpg from
-// images.example.com and optimize it. In apache (always) and nginx (by
-// default) we use a fetcher called "serf". This works fine, but it does run
-// its own event loop. To be more efficient, this is a "native" fetcher that
-// uses nginx's event loop.
-//
-// The fetch is started by the main thread. It will fetch the remote resource
-// from the specific url asynchronously.
-
-#ifndef NET_INSTAWEB_NGX_FETCH_H_
-#define NET_INSTAWEB_NGX_FETCH_H_
-
-extern "C" {
-#include
-#include
-#include
-}
-
-#include "ngx_url_async_fetcher.h"
-#include
-#include "net/instaweb/http/public/url_async_fetcher.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/response_headers.h"
-#include "pagespeed/kernel/http/response_headers_parser.h"
-#include "pagespeed/kernel/thread/pthread_mutex.h"
-
-
-namespace net_instaweb {
-
-typedef bool (*response_handler_pt)(ngx_connection_t* c);
-
-class NgxUrlAsyncFetcher;
-class NgxConnection;
-
-class NgxConnection : public PoolElement {
- public:
- NgxConnection(MessageHandler* handler, int max_keepalive_requests);
- ~NgxConnection();
- void SetSock(u_char *sockaddr, socklen_t socklen) {
- socklen_ = socklen;
- ngx_memcpy(&sockaddr_, sockaddr, socklen);
- }
- // Close ensures that NgxConnection deletes itself at the appropriate time,
- // which can be after receiving a non-keepalive response, or when the remote
- // server closes the connection when the NgxConnection is pooled and idle.
- void Close();
-
- // Once keepalive is disabled, it can't be toggled back on.
- void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
- bool keepalive() { return keepalive_; }
-
- typedef Pool NgxConnectionPool;
-
- static NgxConnection* Connect(ngx_peer_connection_t* pc,
- MessageHandler* handler,
- int max_keepalive_requests);
- static void IdleWriteHandler(ngx_event_t* ev);
- static void IdleReadHandler(ngx_event_t* ev);
- // Terminate will cleanup any idle connections upon shutdown.
- static void Terminate();
-
- static NgxConnectionPool connection_pool;
- static PthreadMutex connection_pool_mutex;
-
- // c_ is owned by NgxConnection and freed in ::Close()
- ngx_connection_t* c_;
- static const int64 keepalive_timeout_ms;
- static const GoogleString ka_header;
-
- private:
- int max_keepalive_requests_;
- bool keepalive_;
- socklen_t socklen_;
- u_char sockaddr_[NGX_SOCKADDRLEN];
- MessageHandler* handler_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxConnection);
-};
-
-class NgxFetch : public PoolElement {
- public:
- NgxFetch(const GoogleString& url,
- AsyncFetch* async_fetch,
- MessageHandler* message_handler,
- ngx_log_t* log);
- ~NgxFetch();
-
- // Start the fetch.
- bool Start(NgxUrlAsyncFetcher* fetcher);
- // Show the completed url, for logging purposes.
- const char* str_url();
- // This fetch task is done. Call Done() on the async_fetch. It will copy the
- // buffer to cache.
- void CallbackDone(bool success);
-
- // Show the bytes received.
- size_t bytes_received();
- void bytes_received_add(int64 x);
- int64 fetch_start_ms();
- void set_fetch_start_ms(int64 start_ms);
- int64 fetch_end_ms();
- void set_fetch_end_ms(int64 end_ms);
- MessageHandler* message_handler();
-
- int get_major_version() {
- return static_cast(status_->http_version / 1000);
- }
- int get_minor_version() {
- return static_cast(status_->http_version % 1000);
- }
- int get_status_code() {
- return static_cast(status_->code);
- }
- ngx_event_t* timeout_event() {
- return timeout_event_;
- }
- void set_timeout_event(ngx_event_t* x) {
- timeout_event_ = x;
- }
- void release_resolver() {
- if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
- ngx_resolve_name_done(resolver_ctx_);
- resolver_ctx_ = NULL;
- }
- }
-
- private:
- response_handler_pt response_handler;
- // Do the initialized work and start the resolver work.
- bool Init();
- bool ParseUrl();
- // Prepare the request and write it to remote server.
- int InitRequest();
- // Create the connection with remote server.
- int Connect();
- void set_response_handler(response_handler_pt handler) {
- response_handler = handler;
- }
- // Only the Static functions could be used in callbacks.
- static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
- // Write the request.
- static void ConnectionWriteHandler(ngx_event_t* wev);
- // Wait for the response.
- static void ConnectionReadHandler(ngx_event_t* rev);
- // Read and parse the first status line.
- static bool HandleStatusLine(ngx_connection_t* c);
- // Read and parse the HTTP headers.
- static bool HandleHeader(ngx_connection_t* c);
- // Read the response body.
- static bool HandleBody(ngx_connection_t* c);
- // Cancel the fetch when it's timeout.
- static void TimeoutHandler(ngx_event_t* tev);
-
- // Add the pagespeed User-Agent.
- void FixUserAgent();
- void FixHost();
-
- const GoogleString str_url_;
- ngx_url_t url_;
- NgxUrlAsyncFetcher* fetcher_;
- AsyncFetch* async_fetch_;
- ResponseHeadersParser parser_;
- MessageHandler* message_handler_;
- int64 bytes_received_;
- int64 fetch_start_ms_;
- int64 fetch_end_ms_;
- bool done_;
- int64 content_length_;
- bool content_length_known_;
-
- struct sockaddr_in sin_;
- ngx_log_t* log_;
- ngx_buf_t* out_;
- ngx_buf_t* in_;
- ngx_pool_t* pool_;
- ngx_http_request_t* r_;
- ngx_http_status_t* status_;
- ngx_event_t* timeout_event_;
- NgxConnection* connection_;
- ngx_resolver_ctx_t* resolver_ctx_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxFetch);
-};
-
-} // namespace net_instaweb
-
-#endif // NET_INSTAWEB_NGX_FETCH_H_
diff --git a/lib/ngx_pagespeed/src/ngx_gzip_setter.cc b/lib/ngx_pagespeed/src/ngx_gzip_setter.cc
deleted file mode 100644
index 6e85bc3..0000000
--- a/lib/ngx_pagespeed/src/ngx_gzip_setter.cc
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: kspoelstra@we-amp.com (Kees Spoelstra)
-
-#include "ngx_gzip_setter.h"
-
-#include
-
-namespace net_instaweb {
-
-NgxGZipSetter g_gzip_setter;
-
-extern "C" {
- // These functions replace the setters for:
- // gzip
- // gzip_types
- // gzip_http_version
- // gzip_vary
- //
- // If these functions are called it means there is an explicit gzip
- // configuration. The gzip configuration set by pagespeed is then rolled
- // back and pagespeed will stop enabling gzip automatically.
- char* ngx_gzip_redirect_conf_set_flag_slot(
- ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- if (g_gzip_setter.enabled()) {
- g_gzip_setter.RollBackAndDisable(cf);
- }
- char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
- return ret;
- }
-
- char* ngx_gzip_redirect_http_types_slot(
- ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- if (g_gzip_setter.enabled()) {
- g_gzip_setter.RollBackAndDisable(cf);
- }
- char* ret = ngx_http_types_slot(cf, cmd, conf);
- return ret;
- }
-
- char* ngx_gzip_redirect_conf_set_enum_slot(
- ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- if (g_gzip_setter.enabled()) {
- g_gzip_setter.RollBackAndDisable(cf);
- }
- char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
- return ret;
- }
- char* ngx_gzip_redirect_conf_set_bitmask_slot(
- ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- if (g_gzip_setter.enabled()) {
- g_gzip_setter.RollBackAndDisable(cf);
- }
- char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
- return ret;
- }
-}
-
-NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
-NgxGZipSetter::~NgxGZipSetter() { }
-
-// Helper functions to determine signature.
-bool HasLocalConfig(ngx_command_t* command) {
- return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
- command->conf == NGX_HTTP_LOC_CONF_OFFSET);
-}
-bool IsNgxFlagCommand(ngx_command_t* command) {
- return (command->set == ngx_conf_set_flag_slot &&
- HasLocalConfig(command));
-}
-bool IsNgxHttpTypesCommand(ngx_command_t* command) {
- return (command->set == ngx_http_types_slot &&
- HasLocalConfig(command));
-}
-bool IsNgxEnumCommand(ngx_command_t* command) {
- return (command->set == ngx_conf_set_enum_slot &&
- HasLocalConfig(command));
-}
-bool IsNgxBitmaskCommand(ngx_command_t* command) {
- return (command->set == ngx_conf_set_bitmask_slot &&
- HasLocalConfig(command));
-}
-
-// Initialize the NgxGzipSetter.
-// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
-// gzip module. Enable if the signature of the zip command matches with what we
-// trust. Also sets up redirects for the configurations. These redirect handle
-// a rollback if expicit configuration is found.
-// If commands are not found the method will inform the user by logging.
-void NgxGZipSetter::Init(ngx_conf_t* cf) {
-#if (NGX_HTTP_GZIP)
- bool gzip_signature_mismatch = false;
- bool other_signature_mismatch = false;
- for (int m = 0; ngx_modules[m] != NULL; m++) {
- if (ngx_modules[m]->commands != NULL) {
- for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
- ngx_command_t* current_command =& ngx_modules[m]->commands[c];
-
- // We look for the gzip command, and the exact signature we trust
- // this means configured as an config location offset
- // and a ngx_flag_t setter.
- // Also see:
- // ngx_conf_handler in ngx_conf_file.c
- // ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
- if (gzip_command_.command_ == NULL &&
- STR_EQ_LITERAL(current_command->name, "gzip")) {
- if (IsNgxFlagCommand(current_command)) {
- current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
- gzip_command_.command_ = current_command;
- gzip_command_.module_ = ngx_modules[m];
- enabled_ = 1;
- } else {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0,
- "pagespeed: cannot set gzip, signature mismatch");
- gzip_signature_mismatch = true;
- }
- }
-
- if (!gzip_http_version_command_.command_ &&
- STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
- if (IsNgxEnumCommand(current_command)) {
- current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
- gzip_http_version_command_.command_ = current_command;
- gzip_http_version_command_.module_ = ngx_modules[m];
- } else {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0,
- "pagespeed: cannot set gzip_http_version, signature mismatch");
- other_signature_mismatch = true;
- }
- }
-
- if (!gzip_proxied_command_.command_ &&
- STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
- if (IsNgxBitmaskCommand(current_command)) {
- current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
- gzip_proxied_command_.command_ = current_command;
- gzip_proxied_command_.module_ = ngx_modules[m];
- } else {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0,
- "pagespeed: cannot set gzip_proxied, signature mismatch");
- other_signature_mismatch = true;
- }
- }
-
- if (!gzip_http_types_command_.command_ &&
- STR_EQ_LITERAL(current_command->name, "gzip_types")) {
- if (IsNgxHttpTypesCommand(current_command)) {
- current_command->set = ngx_gzip_redirect_http_types_slot;
- gzip_http_types_command_.command_ = current_command;
- gzip_http_types_command_.module_ = ngx_modules[m];
- } else {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0,
- "pagespeed: cannot set gzip_types, signature mismatch");
- other_signature_mismatch = true;
- }
- }
-
- if (!gzip_vary_command_.command_ &&
- STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
- if (IsNgxFlagCommand(current_command)) {
- current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
- gzip_vary_command_.command_ = current_command;
- gzip_vary_command_.module_ = ngx_modules[m];
- } else {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0,
- "pagespeed: cannot set gzip_vary, signature mismatch");
- other_signature_mismatch = true;
- }
- }
- }
- }
- }
- if (gzip_signature_mismatch) {
- return; // Already logged error.
- } else if (!enabled_) {
- // Looked through all the available commands and didn't find the "gzip" one.
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
- return;
- } else if (other_signature_mismatch) {
- return; // Already logged error.
- } else if (!gzip_vary_command_.command_) {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
- return;
- } else if (!gzip_http_types_command_.command_) {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
- return;
- } else if (!gzip_http_version_command_.command_) {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
- return;
- } else if (!gzip_proxied_command_.command_) {
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
- return;
- } else {
- return; // Success.
- }
-#else
- ngx_conf_log_error(
- NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
- return;
-#endif
-}
-
-void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
- return GetModuleConfPtr(cf) + command_->offset;
-}
-
-char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
- return reinterpret_cast(
- ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
-}
-
-void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_flag_t value) {
- ngx_flag_t* flag = reinterpret_cast(command_ctx->GetConfPtr(cf));
- *flag = value;
- // Save the flag position for possible rollback.
- ngx_flags_set_.push_back(flag);
-}
-
-void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_uint_t value) {
- ngx_uint_t* enum_to_set =
- reinterpret_cast(command_ctx->GetConfPtr(cf));
- *enum_to_set = value;
- ngx_uint_set_.push_back(enum_to_set);
-}
-
-void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_uint_t value) {
- ngx_uint_t* enum_to_set =
- reinterpret_cast(command_ctx->GetConfPtr(cf));
- *enum_to_set = value;
- ngx_uint_set_.push_back(enum_to_set);
-}
-
-// These are the content types we want to compress.
-ngx_str_t gzip_http_types[] = {
- ngx_string("application/ecmascript"),
- ngx_string("application/javascript"),
- ngx_string("application/json"),
- ngx_string("application/pdf"),
- ngx_string("application/postscript"),
- ngx_string("application/x-javascript"),
- ngx_string("image/svg+xml"),
- ngx_string("text/css"),
- ngx_string("text/csv"),
- // ngx_string("text/html"), // This is the default implied value.
- ngx_string("text/javascript"),
- ngx_string("text/plain"),
- ngx_string("text/xml"),
- ngx_null_string // Indicates end of array.
-};
-
-gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
- bool value) {
- if (!enabled_) {
- return kEnableGZipNotEnabled;
- }
- if (gzip_command_.command_) {
- SetNgxConfFlag(cf, &gzip_command_, value);
- }
- return kEnableGZipOk;
-}
-
-void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
- if (!enabled_) {
- return;
- }
-
- // When we get called twice for the same location{}, we ignore the second call
- // to prevent adding duplicate gzip http types and so on.
- ngx_flag_t* flag =
- reinterpret_cast(gzip_command_.GetConfPtr(cf));
- if (*flag == 1) {
- return;
- }
- SetGZipForLocation(cf, true);
- if (gzip_vary_command_.command_) {
- SetNgxConfFlag(cf, &gzip_vary_command_, 1);
- }
- if (gzip_http_version_command_.command_) {
- SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
- }
- if (gzip_proxied_command_.command_) {
- SetNgxConfBitmask(
- cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
- }
-
- // This is actually the most prone to future API changes, because gzip_types
- // is not a simple type like ngx_flag_t. The signature check should be enough
- // to prevent problems.
- AddGZipHTTPTypes(cf);
- return;
-}
-
-void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
- if (gzip_http_types_command_.command_) {
- // Following should not happen, but if it does return gracefully.
- if (cf->args->nalloc < 2) {
- ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
- "pagespeed: unexpected small cf->args in gzip_types");
- return;
- }
-
- ngx_command_t* command = gzip_http_types_command_.command_;
- char* gzip_conf = reinterpret_cast(
- gzip_http_types_command_.GetModuleConfPtr(cf));
-
- // Backup the old settings.
- ngx_str_t old_elt0 = reinterpret_cast(cf->args->elts)[0];
- ngx_str_t old_elt1 = reinterpret_cast(cf->args->elts)[1];
- ngx_uint_t old_nelts = cf->args->nelts;
-
- // Setup first arg.
- ngx_str_t gzip_types_string = ngx_string("gzip_types");
- reinterpret_cast(cf->args->elts)[0] = gzip_types_string;
- cf->args->nelts = 2;
-
- ngx_str_t* http_types = gzip_http_types;
- while (http_types->data) {
- ngx_str_t d;
- // We allocate the http type on the configuration pool and actually
- // leak this if we rollback. This does not seem to be a big problem,
- // because nginx also allocates tokens in ngx_conf_file.c and does not
- // free them. This way they can be used safely by configurations.
- // We must use a copy of gzip_http_types array here because nginx will
- // manipulate the values.
- // TODO(kspoelstra): better would be to allocate once on init and not
- // every time we enable gzip. This needs further investigation, sharing
- // tokens might be problematic.
- // For now I think it is not a large problem. This might add up in case
- // of a large multi server/location config with a lot of "pagespeed on"
- // directives.
- // Estimates are 300-400KB for 1000 times "pagespeed on".
- d.data = reinterpret_cast(
- ngx_pnalloc(cf->pool, http_types->len + 1));
- snprintf(reinterpret_cast(d.data), http_types->len + 1, "%s",
- reinterpret_cast(http_types->data));
- d.len = http_types->len;
- reinterpret_cast(cf->args->elts)[1] = d;
- // Call the original setter.
- ngx_http_types_slot(cf, command, gzip_conf);
- http_types++;
- }
-
- // Restore args.
- cf->args->nelts = old_nelts;
- reinterpret_cast(cf->args->elts)[1] = old_elt1;
- reinterpret_cast(cf->args->elts)[0] = old_elt0;
-
- // Backup configuration location for rollback.
- ngx_httptypes_set_.push_back(gzip_conf + command->offset);
- }
-}
-
-void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
- ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
- "pagespeed: rollback gzip, explicit configuration");
- for (std::vector::iterator i = ngx_flags_set_.begin();
- i != ngx_flags_set_.end(); ++i) {
- *(*i)=NGX_CONF_UNSET;
- }
- for (std::vector::iterator i = ngx_uint_set_.begin();
- i != ngx_uint_set_.end(); ++i) {
- *(*i)=NGX_CONF_UNSET_UINT;
- }
- for (std::vector::iterator i = ngx_httptypes_set_.begin();
- i != ngx_httptypes_set_.end(); ++i) {
- ngx_array_t** type_array = reinterpret_cast(*i);
- ngx_array_destroy(*type_array);
- *type_array = NULL;
- }
- enabled_ = 0;
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_gzip_setter.h b/lib/ngx_pagespeed/src/ngx_gzip_setter.h
deleted file mode 100644
index 6d0d698..0000000
--- a/lib/ngx_pagespeed/src/ngx_gzip_setter.h
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: kspoelstra@we-amp.com (Kees Spoelstra)
-
-/*
- * NgxGZipSetter sets up gzip for pagespeed
- * with the following configuration:
- * gzip on;
- * gzip_vary on;
- * gzip_types application/ecmascript;
- * gzip_types application/javascript;
- * gzip_types application/json;
- * gzip_types application/pdf;
- * gzip_types application/postscript;
- * gzip_types application/x-javascript;
- * gzip_types image/svg+xml;
- * gzip_types text/css;
- * gzip_types text/csv;
- * gzip_types text/javascript;
- * gzip_types text/plain;
- * gzip_types text/xml;
- * gzip_http_version 1.0;
- *
- * If there is an explicit gzip configuration in the nginx.conf
- * pagespeed will rollback the set configuration and let the
- * user decide what the configuration will be.
- *
- * It manipulates the configuration by manipulating ngx_flag_t
- * and ngx_uint_t settings directly and using the nginx setter for
- * gzip_http_types.
- * This is probably a safe way to do it. If this mechanism
- * changes all non nginx module setup & configuration will
- * fail.
- */
-
-#ifndef NGX_GZIP_SETTER_H_
-#define NGX_GZIP_SETTER_H_
-
-extern "C" {
-#include
-#include
-#include
-}
-
-#include
-
-#include "ngx_pagespeed.h"
-
-#include "pagespeed/kernel/base/basictypes.h"
-
-namespace net_instaweb {
-
-// We need this class because configuration for gzip is in different modules, so
-// just saving the command will not work.
-class ngx_command_ctx {
- public:
- ngx_command_ctx() : command_(NULL), module_(NULL) { }
- void* GetConfPtr(ngx_conf_t* cf);
- char* GetModuleConfPtr(ngx_conf_t* cf);
- ngx_command_t* command_;
- ngx_module_t* module_;
-};
-
-enum gzs_enable_result {
- kEnableGZipOk,
- kEnableGZipPartial,
- kEnableGZipNotEnabled
-};
-
-class NgxGZipSetter {
- std::vector ngx_flags_set_;
- std::vector ngx_uint_set_;
- std::vector ngx_httptypes_set_;
- ngx_command_ctx gzip_command_;
- ngx_command_ctx gzip_http_types_command_;
- ngx_command_ctx gzip_proxied_command_;
- ngx_command_ctx gzip_vary_command_;
- ngx_command_ctx gzip_http_version_command_;
- bool enabled_;
-
- public:
- NgxGZipSetter();
- ~NgxGZipSetter();
- void Init(ngx_conf_t* cf);
-
- void SetNgxConfFlag(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_flag_t value);
- void SetNgxConfEnum(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_uint_t value);
- void SetNgxConfBitmask(ngx_conf_t* cf,
- ngx_command_ctx* command_ctx,
- ngx_uint_t value);
- void EnableGZipForLocation(ngx_conf_t* cf);
- gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
- void AddGZipHTTPTypes(ngx_conf_t* cf);
- void RollBackAndDisable(ngx_conf_t* cf);
-
- bool enabled() { return enabled_; }
-
- private:
- DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
-};
-
-extern NgxGZipSetter g_gzip_setter;
-
-} // namespace net_instaweb
-
-#endif // NGX_GZIP_SETTER_H_
diff --git a/lib/ngx_pagespeed/src/ngx_list_iterator.cc b/lib/ngx_pagespeed/src/ngx_list_iterator.cc
deleted file mode 100644
index 53e2799..0000000
--- a/lib/ngx_pagespeed/src/ngx_list_iterator.cc
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#include "ngx_list_iterator.h"
-
-namespace net_instaweb {
-
-NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
- part_(part),
- index_within_part_(0) {}
-
-ngx_table_elt_t* NgxListIterator::Next() {
- if (index_within_part_ >= part_->nelts) {
- if (part_->next == NULL) {
- return NULL;
- }
- part_ = part_->next;
- index_within_part_ = 0;
- }
- ngx_table_elt_t* elts = static_cast(part_->elts);
- return &elts[index_within_part_++]; // Intentional post-increment.
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_list_iterator.h b/lib/ngx_pagespeed/src/ngx_list_iterator.h
deleted file mode 100644
index 11b3383..0000000
--- a/lib/ngx_pagespeed/src/ngx_list_iterator.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-//
-// Simplifies iteration over nginx lists.
-//
-//
-
-#ifndef NGX_LIST_ITERATOR_H_
-#define NGX_LIST_ITERATOR_H_
-
-extern "C" {
-#include
-}
-
-#include "base/basictypes.h"
-
-namespace net_instaweb {
-
-class NgxListIterator {
- public:
- explicit NgxListIterator(ngx_list_part_t* part);
-
- // Return the next element of the list if there is one, NULL otherwise.
- ngx_table_elt_t* Next();
-
- private:
- ngx_list_part_t* part_;
- ngx_uint_t index_within_part_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_LIST_ITERATOR_H_
diff --git a/lib/ngx_pagespeed/src/ngx_message_handler.cc b/lib/ngx_pagespeed/src/ngx_message_handler.cc
deleted file mode 100644
index edf0168..0000000
--- a/lib/ngx_pagespeed/src/ngx_message_handler.cc
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2013 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Author: oschaaf@gmail.com (Otto van der Schaaf)
-
-#include "ngx_message_handler.h"
-
-#include
-
-#include "net/instaweb/public/version.h"
-#include "pagespeed/kernel/base/abstract_mutex.h"
-#include "pagespeed/kernel/base/debug.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/time_util.h"
-#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
-
-namespace {
-
-// This will be prefixed to every logged message.
-const char kModuleName[] = "ngx_pagespeed";
-
-// If set, the crash handler will use this to output a backtrace using
-// ngx_log_error.
-ngx_log_t* global_log = NULL;
-
-} // namespace
-
-extern "C" {
- static void signal_handler(int sig) {
- // Try to output the backtrace to the log file. Since this may end up
- // crashing/deadlocking/etc. we set an alarm() to abort us if it comes to
- // that.
- alarm(2);
- if (global_log != NULL) {
- ngx_log_error(NGX_LOG_ALERT, global_log, 0, "Trapped signal [%d]\n%s",
- sig, net_instaweb::StackTraceString().c_str());
- } else {
- fprintf(stderr, "Trapped signal [%d]\n%s\n",
- sig, net_instaweb::StackTraceString().c_str());
- }
- kill(getpid(), SIGKILL);
- }
-} // extern "C"
-
-namespace net_instaweb {
-
-NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
- : SystemMessageHandler(timer, mutex),
- log_(NULL) {
-}
-
-// Installs a signal handler for common crash signals, that tries to print
-// out a backtrace.
-void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
- global_log = log;
- signal(SIGTRAP, signal_handler); // On check failures
- signal(SIGABRT, signal_handler);
- signal(SIGFPE, signal_handler);
- signal(SIGSEGV, signal_handler);
-}
-
-ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
- switch (type) {
- case kInfo:
- return NGX_LOG_INFO;
- case kWarning:
- return NGX_LOG_WARN;
- case kError:
- return NGX_LOG_ERR;
- case kFatal:
- return NGX_LOG_ALERT;
- }
-
- // This should never fall through, but some compilers seem to complain if
- // we don't include this.
- return NGX_LOG_ALERT;
-}
-
-void NgxMessageHandler::MessageSImpl(MessageType type,
- const GoogleString& message) {
- if (log_ != NULL) {
- ngx_uint_t log_level = GetNgxLogLevel(type);
- ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
- kModuleName, kModPagespeedVersion, message.c_str());
- } else {
- GoogleMessageHandler::MessageSImpl(type, message);
- }
- AddMessageToBuffer(type, message);
-}
-
-void NgxMessageHandler::FileMessageSImpl(
- MessageType type, const char* file, int line, const GoogleString& message) {
- if (log_ != NULL) {
- ngx_uint_t log_level = GetNgxLogLevel(type);
- ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
- kModuleName, kModPagespeedVersion, file, line,
- message.c_str());
- } else {
- GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
- }
- AddMessageToBuffer(type, file, line, message);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_message_handler.h b/lib/ngx_pagespeed/src/ngx_message_handler.h
deleted file mode 100644
index 78ccfc9..0000000
--- a/lib/ngx_pagespeed/src/ngx_message_handler.h
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright 2013 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Author: oschaaf@gmail.com (Otto van der Schaaf)
-
-#ifndef NGX_MESSAGE_HANDLER_H_
-#define NGX_MESSAGE_HANDLER_H_
-
-extern "C" {
- #include
-#if (NGX_THREADS)
- #include
-#endif
- #include
- #include
-}
-
-#include
-
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/system/system_message_handler.h"
-
-namespace net_instaweb {
-
-class AbstractMutex;
-class Timer;
-
-// Implementation of a message handler that uses ngx_log_error()
-// logging to emit messages, with a fallback to GoogleMessageHandler
-class NgxMessageHandler : public SystemMessageHandler {
- public:
- explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
-
- // Installs a signal handler for common crash signals that tries to print
- // out a backtrace.
- static void InstallCrashHandler(ngx_log_t* log);
-
- void set_log(ngx_log_t* log) { log_ = log; }
- ngx_log_t* log() { return log_; }
-
- protected:
- virtual void MessageSImpl(MessageType type, const GoogleString& message);
-
- virtual void FileMessageSImpl(MessageType type, const char* file,
- int line, const GoogleString& message);
-
- private:
- ngx_uint_t GetNgxLogLevel(MessageType type);
- ngx_log_t* log_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_MESSAGE_HANDLER_H_
diff --git a/lib/ngx_pagespeed/src/ngx_pagespeed.cc b/lib/ngx_pagespeed/src/ngx_pagespeed.cc
deleted file mode 100644
index 579832f..0000000
--- a/lib/ngx_pagespeed/src/ngx_pagespeed.cc
+++ /dev/null
@@ -1,3291 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-/*
- * Usage:
- * server {
- * pagespeed on|off;
- * }
- */
-
-#include "ngx_pagespeed.h"
-
-#include
-#include
-
-#include "ngx_base_fetch.h"
-#include "ngx_caching_headers.h"
-#include "ngx_gzip_setter.h"
-#include "ngx_list_iterator.h"
-#include "ngx_message_handler.h"
-#include "ngx_rewrite_driver_factory.h"
-#include "ngx_rewrite_options.h"
-#include "ngx_server_context.h"
-
-#include "net/instaweb/http/public/async_fetch.h"
-#include "net/instaweb/http/public/cache_url_async_fetcher.h"
-#include "net/instaweb/http/public/request_context.h"
-#include "net/instaweb/public/global_constants.h"
-#include "net/instaweb/public/version.h"
-#include "net/instaweb/rewriter/public/experiment_matcher.h"
-#include "net/instaweb/rewriter/public/experiment_util.h"
-#include "net/instaweb/rewriter/public/process_context.h"
-#include "net/instaweb/rewriter/public/resource_fetch.h"
-#include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "net/instaweb/rewriter/public/rewrite_query.h"
-#include "net/instaweb/rewriter/public/rewrite_stats.h"
-#include "net/instaweb/rewriter/public/static_asset_manager.h"
-#include "net/instaweb/util/public/fallback_property_page.h"
-#include "pagespeed/automatic/proxy_fetch.h"
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/null_message_handler.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/base/stack_buffer.h"
-#include "pagespeed/kernel/base/stdio_file_system.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/string_writer.h"
-#include "pagespeed/kernel/base/time_util.h"
-#include "pagespeed/kernel/http/content_type.h"
-#include "pagespeed/kernel/http/google_url.h"
-#include "pagespeed/kernel/http/query_params.h"
-#include "pagespeed/kernel/html/html_keywords.h"
-#include "pagespeed/kernel/thread/pthread_shared_mem.h"
-#include "pagespeed/kernel/util/gzip_inflater.h"
-#include "pagespeed/kernel/util/statistics_logger.h"
-#include "pagespeed/system/in_place_resource_recorder.h"
-#include "pagespeed/system/system_caches.h"
-#include "pagespeed/system/system_request_context.h"
-#include "pagespeed/system/system_rewrite_options.h"
-#include "pagespeed/system/system_server_context.h"
-#include "pagespeed/system/system_thread_system.h"
-
-extern ngx_module_t ngx_pagespeed;
-
-// Unused flag, see
-// http://lxr.evanmiller.org/http/source/http/ngx_http_request.h#L130
-#define NGX_HTTP_PAGESPEED_BUFFERED 0x08
-#define POST_BUF_READ_SIZE 65536
-
-// Needed for SystemRewriteDriverFactory to use shared memory.
-#define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
-#define NGINX_1_13_4 1013004
-
-net_instaweb::NgxRewriteDriverFactory* active_driver_factory = NULL;
-
-namespace net_instaweb {
-
-const char* kInternalEtagName = "@psol-etag";
-// The process context takes care of proactively initialising
-// a few libraries for us, some of which are not thread-safe
-// when they are initialized lazily.
-ProcessContext* process_context = new ProcessContext();
-bool process_context_cleanup_hooked = false;
-
-StringPiece str_to_string_piece(ngx_str_t s) {
- return StringPiece(reinterpret_cast(s.data), s.len);
-}
-
-// Nginx uses memory pools, like Apache, allocating strings a request needs from
-// the pool and then throwing it all away when the request finishes. Use this
-// when you need to pass a short string to nginx and want it to take ownership
-// of the string.
-char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
- // Need space for the final null.
- ngx_uint_t buffer_size = sp.size() + 1;
- char* s = static_cast(ngx_palloc(pool, buffer_size));
- if (s == NULL) {
- LOG(ERROR) << "string_piece_to_pool_string: ngx_palloc() returned NULL";
- DCHECK(false);
- return NULL;
- }
- sp.copy(s, buffer_size /* max to copy */);
- s[buffer_size-1] = '\0'; // Null terminate it.
- return s;
-}
-
-// When passing the body of http responses between filters Nginx uses a linked
-// list of buffers ("buffer chain"), again like Apache. This constructs one of
-// those lists from a StringPiece. This is what you use when you need to pass a
-// (potentially) longer string to nginx and want it to take ownership.
-ngx_int_t string_piece_to_buffer_chain(
- ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
- bool send_last_buf, bool send_flush) {
- // Below, *link_ptr will be NULL if we're starting the chain, and the head
- // chain link.
- *link_ptr = NULL;
-
- // If non-null, the current last link in the chain.
- ngx_chain_t* tail_link = NULL;
-
- // How far into sp we're currently working on.
- ngx_uint_t offset;
-
- // Other modules seem to default to ngx_pagesize.
- ngx_uint_t max_buffer_size = ngx_pagesize;
- for (offset = 0 ;
- offset < sp.size() ||
- // If we need to send the last buffer bit and there's no data, we
- // should send a single empty buffer. Otherwise we shouldn't
- // generate empty buffers.
- (offset == 0 && sp.size() == 0);
- offset += max_buffer_size) {
- // Prepare a new nginx buffer to put our buffered writes into.
- ngx_buf_t* b = static_cast(ngx_calloc_buf(pool));
- if (b == NULL) {
- return NGX_ERROR;
- }
-
- if (sp.size() == 0) {
- CHECK(offset == 0); // NOLINT
- b->pos = b->start = b->end = b->last = NULL;
- // The purpose of this buffer is just to pass along last_buf.
- b->sync = 1;
- } else {
- CHECK(sp.size() > offset);
- ngx_uint_t b_size = sp.size() - offset;
- if (b_size > max_buffer_size) {
- b_size = max_buffer_size;
- }
-
- b->start = b->pos = static_cast(ngx_palloc(pool, b_size));
- if (b->pos == NULL) {
- return NGX_ERROR;
- }
-
- // Copy our writes over. We're copying from sp[offset] up to
- // sp[offset + b_size] into b which has size b_size.
- sp.copy(reinterpret_cast(b->pos), b_size, offset);
- b->last = b->end = b->pos + b_size;
-
- b->temporary = 1; // Identify this buffer as in-memory and mutable.
- }
-
- // Prepare a chain link.
- ngx_chain_t* cl = static_cast(ngx_alloc_chain_link(pool));
- if (cl == NULL) {
- return NGX_ERROR;
- }
-
- cl->buf = b;
- cl->next = NULL;
-
- if (*link_ptr == NULL) {
- // This is the first link in the returned chain.
- *link_ptr = cl;
- } else {
- // Link us into the chain.
- CHECK(tail_link != NULL);
- tail_link->next = cl;
- }
-
- tail_link = cl;
- }
-
-
- CHECK(tail_link != NULL);
- if (send_flush) {
- tail_link->buf->flush = true;
- }
- if (send_last_buf) {
- tail_link->buf->last_buf = true;
- }
-
- return NGX_OK;
-}
-
-// Get the context for this request. ps_connection_read_handler should already
-// have been called to create it.
-ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r) {
- return static_cast(
- ngx_http_get_module_ctx(r, ngx_pagespeed));
-}
-
-// Tell nginx whether we have network activity we're waiting for so that it sets
-// a write handler. See src/http/ngx_http_request.c:2083.
-void ps_set_buffered(ngx_http_request_t* r, bool on) {
- if (on) {
- r->buffered |= NGX_HTTP_PAGESPEED_BUFFERED;
- } else {
- r->buffered &= ~NGX_HTTP_PAGESPEED_BUFFERED;
- }
-}
-
-namespace {
-
-void ps_release_base_fetch(ps_request_ctx_t* ctx);
-
-} // namespace
-
-namespace ps_base_fetch {
-
-ngx_http_output_header_filter_pt ngx_http_next_header_filter;
-ngx_http_output_body_filter_pt ngx_http_next_body_filter;
-
-ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- if (r->header_only) {
- return NGX_OK;
- }
- if (ctx == NULL || ctx->base_fetch == NULL) {
- return ngx_http_next_body_filter(r, in);
- }
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http pagespeed write filter \"%V\"", &r->uri);
-
- // send response body
- if (in || r->connection->buffered) {
- ngx_int_t rc = ngx_http_next_body_filter(r, in);
- // We can't indicate that we are done yet, because we have an active base
- // fetch associated to this request.
- if (rc != NGX_OK) {
- return rc;
- }
- }
-
- return NGX_AGAIN;
-}
-
-// This runs on the nginx event loop in response to seeing the byte PageSpeed
-// sent over the pipe to trigger the nginx-side code. Copy whatever is ready
-// from PageSpeed out to the browser (headers and/or body).
-ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- ngx_int_t rc;
- ngx_chain_t* cl = NULL;
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ps fetch handler: %V", &r->uri);
-
- if (ngx_terminate || ngx_exiting) {
- ps_set_buffered(r, false);
- ps_release_base_fetch(ctx);
- return NGX_ERROR;
- }
-
- if (!r->header_sent) {
- int status_code = ctx->base_fetch->response_headers()->status_code();
- bool status_ok = (status_code != 0) && (status_code < 400);
-
- // Pass on error handling for non-success status codes to nginx for
- // responses where PSOL acts as a content generator (pagespeed resource,
- // ipro hit, admin pages).
- // This generates nginx's default error responses, but also allows header
- // modules running after us to manipulate those responses.
- if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
- && ctx->base_fetch->base_fetch_type() != kIproLookup)) {
- ps_release_base_fetch(ctx);
- ngx_http_filter_finalize_request(r, NULL, status_code);
- return NGX_DONE;
- }
-
- if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
- ngx_table_elt_t* header;
- NgxListIterator it(&(r->headers_out.headers.part));
- while ((header = it.Next()) != NULL) {
- // We need to remember a few headers when ModifyCachingHeaders is off,
- // so we can send them unmodified in copy_response_headers_to_ngx().
- // This just sets the hash to 0 for all other headers. That way, we
- // avoid some relatively complicated code to reconstruct these headers.
- if (!(STR_CASE_EQ_LITERAL(header->key, "Cache-Control") ||
- (ctx->preserve_caching_headers == kPreserveAllCachingHeaders &&
- (STR_CASE_EQ_LITERAL(header->key, "Etag") ||
- STR_CASE_EQ_LITERAL(header->key, "Date") ||
- STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
- STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
- header->hash = 0;
- if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
- // There's a possible issue with the location header, where setting
- // the hash to 0 is not enough. See:
- // https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
- r->headers_out.location = NULL;
- }
- }
- }
- } else {
- ngx_http_clean_header(r);
- }
- // collect response headers from pagespeed
- rc = ctx->base_fetch->CollectHeaders(&r->headers_out);
- if (rc == NGX_ERROR) {
- ps_release_base_fetch(ctx);
- return NGX_HTTP_INTERNAL_SERVER_ERROR;
- }
-
- // send response headers
- rc = ngx_http_next_header_filter(r);
-
- // standard nginx send header check see ngx_http_send_response
- if (rc == NGX_ERROR || rc > NGX_OK) {
- ps_release_base_fetch(ctx);
- return ngx_http_filter_finalize_request(r, NULL, rc);
- }
-
- // for in_place_check_header_filter
- if (rc < NGX_OK && rc != NGX_AGAIN) {
- CHECK(rc == NGX_DONE);
- return NGX_DONE;
- }
-
- ps_set_buffered(r, true);
- }
-
- // collect response body from pagespeed
- // Pass the optimized content along to later body filters.
- // From Weibin: This function should be called mutiple times. Store the
- // whole file in one chain buffers is too aggressive. It could consume
- // too much memory in busy servers.
-
- rc = ctx->base_fetch->CollectAccumulatedWrites(&cl);
- ngx_log_error(NGX_LOG_DEBUG, ctx->r->connection->log, 0,
- "CollectAccumulatedWrites, %d", rc);
-
- if (rc == NGX_ERROR) {
- ps_set_buffered(r, false);
- ps_release_base_fetch(ctx);
- return NGX_HTTP_INTERNAL_SERVER_ERROR;
- }
-
- if (rc == NGX_AGAIN && cl == NULL) {
- // there is no body buffer to send now.
- return NGX_AGAIN;
- }
-
- if (rc == NGX_OK) {
- ps_set_buffered(r, false);
- ps_release_base_fetch(ctx);
- }
-
- return ps_base_fetch_filter(r, cl);
-}
-
-void ps_base_fetch_filter_init() {
- ngx_http_next_header_filter = ngx_http_top_header_filter;
- ngx_http_next_body_filter = ngx_http_top_body_filter;
- ngx_http_top_body_filter = ps_base_fetch_filter;
-}
-
-} // namespace ps_base_fetch
-
-
-namespace {
-
-// Setting headers in nginx is tricky because it's not just a matter of adding
-// them to a list. You also need to remove them if there's already one there,
-// as well as setting the shortcut pointers (both upper case and lower case).
-//
-// Based on ngx_http_add_cache_control.
-ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
- // First strip existing cache-control headers.
- ngx_table_elt_t* header;
- NgxListIterator it(&(r->headers_out.headers.part));
- while ((header = it.Next()) != NULL) {
- if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
- // Response headers with hash of 0 are excluded from the response.
- header->hash = 0;
- }
- }
-
- // Now add our new cache control header.
- if (r->headers_out.cache_control.elts == NULL) {
- ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
- 1, sizeof(ngx_table_elt_t*));
- if (rc != NGX_OK) {
- return NGX_ERROR;
- }
- }
- ngx_table_elt_t** cache_control_headers = static_cast(
- ngx_array_push(&r->headers_out.cache_control));
- if (cache_control_headers == NULL) {
- return NGX_ERROR;
- }
- cache_control_headers[0] = static_cast(
- ngx_list_push(&r->headers_out.headers));
- if (cache_control_headers[0] == NULL) {
- return NGX_ERROR;
- }
- cache_control_headers[0]->hash = 1;
- ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
- cache_control_headers[0]->value.len = strlen(cache_control);
- cache_control_headers[0]->value.data =
- reinterpret_cast(cache_control);
-
- return NGX_OK;
-}
-
-// Returns false if the header wasn't found. Otherwise sets cache_control and
-// returns true;
-bool ps_get_cache_control(ngx_http_request_t* r, GoogleString* cache_control) {
- // Use headers_out.cache_control instead of looking for Cache-Control in
- // headers_out.headers, because if an upstream sent multiple Cache-Control
- // headers they're already combined in headers_out.cache_control.
- auto ccp = static_cast(r->headers_out.cache_control.elts);
- if (ccp == nullptr) {
- return false; // Header not present.
- }
- bool first_segment = true;
- for (ngx_uint_t i = 0; i < r->headers_out.cache_control.nelts; i++) {
- if (ccp[i]->hash == 0) {
- continue; // Elements with a hash of 0 are marked as excluded.
- }
- if (first_segment) {
- first_segment = false;
- } else {
- cache_control->append(", ");
- }
- cache_control->append(reinterpret_cast(ccp[i]->value.data),
- ccp[i]->value.len);
- }
- return true;
-}
-
-template
-void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
- // Standard nginx idiom for iterating over a list. See ngx_list.h
- ngx_uint_t i;
- const ngx_list_part_t* part = &from.part;
- const ngx_table_elt_t* header = static_cast(part->elts);
-
- for (i = 0 ; /* void */; i++) {
- if (i >= part->nelts) {
- if (part->next == NULL) {
- break;
- }
-
- part = part->next;
- header = static_cast(part->elts);
- i = 0;
- }
- // Make sure we don't copy over headers that are unset.
- if (header[i].hash == 0) {
- continue;
- }
- StringPiece key = str_to_string_piece(header[i].key);
- StringPiece value = str_to_string_piece(header[i].value);
-
- to->Add(key, value);
- }
-}
-} // namespace
-
-void copy_response_headers_from_ngx(const ngx_http_request_t* r,
- ResponseHeaders* headers) {
- headers->set_major_version(r->http_version / 1000);
- headers->set_minor_version(r->http_version % 1000);
- copy_headers_from_table(r->headers_out.headers, headers);
-
- headers->set_status_code(r->headers_out.status);
-
- // Manually copy over the content type because it's not included in
- // request_->headers_out.headers.
- headers->Add(HttpAttributes::kContentType,
- str_to_string_piece(r->headers_out.content_type));
-
- // When we don't have a date header, set one with the current time.
- if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
- PosixTimer timer;
- headers->SetDate(timer.NowMs());
- }
-
- // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
- headers->ComputeCaching();
-}
-
-void copy_request_headers_from_ngx(const ngx_http_request_t* r,
- RequestHeaders* headers) {
- // TODO(chaizhenhua): only allow RewriteDriver::kPassThroughRequestAttributes?
- headers->set_major_version(r->http_version / 1000);
- headers->set_minor_version(r->http_version % 1000);
- copy_headers_from_table(r->headers_in.headers, headers);
-}
-
-// PSOL produces caching headers that need some changes before we can send them
-// out. Make those changes and populate r->headers_out from pagespeed_headers.
-ngx_int_t copy_response_headers_to_ngx(
- ngx_http_request_t* r,
- const ResponseHeaders& pagespeed_headers,
- PreserveCachingHeaders preserve_caching_headers) {
- ngx_http_headers_out_t* headers_out = &r->headers_out;
- headers_out->status = pagespeed_headers.status_code();
-
- ngx_int_t i;
- for (i = 0 ; i < pagespeed_headers.NumAttributes() ; i++) {
- // For IPRO cache misses, these gs_ variables may point to freed memory
- // when nginx writes the headers to the output as the NgxBaseFetch instance
- // that owns this memory gets released during request processing. So we
- // copy these strings later on.
- const GoogleString& name_gs = pagespeed_headers.Name(i);
- const GoogleString& value_gs = pagespeed_headers.Value(i);
-
- if (preserve_caching_headers == kPreserveAllCachingHeaders) {
- if (StringCaseEqual(name_gs, "ETag") ||
- StringCaseEqual(name_gs, "Expires") ||
- StringCaseEqual(name_gs, "Date") ||
- StringCaseEqual(name_gs, "Last-Modified") ||
- StringCaseEqual(name_gs, "Cache-Control")) {
- continue;
- }
- } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
- // Retain the original Cache-Control header, but send the recomputed
- // values for all other cache-related headers.
- if (StringCaseEqual(name_gs, "Cache-Control")) {
- continue;
- }
- } // else we don't preserve any headers.
-
- ngx_str_t name, value;
- value.len = value_gs.size();
- value.data = reinterpret_cast(
- string_piece_to_pool_string(r->pool, value_gs.c_str()));
-
- // To prevent the gzip module from clearing weak etags, we output them
- // using a different name here. The etag header filter module runs behind
- // the gzip compressors header filter, and will rename it to 'ETag'
- if (StringCaseEqual(name_gs, "etag")
- && StringCaseStartsWith(value_gs, "W/")) {
- name.len = strlen(kInternalEtagName);
- name.data = reinterpret_cast(
- const_cast(kInternalEtagName));
- } else {
- name.len = name_gs.size();
- name.data = reinterpret_cast(
- string_piece_to_pool_string(r->pool, name_gs.c_str()));
- }
-
- // In case string_piece_to_pool_string failed:
- if (name.data == NULL || value.data == NULL) {
- return NGX_ERROR;
- }
-
- // TODO(jefftk): If we're setting a cache control header we'd like to
- // prevent any downstream code from changing it. Specifically, if we're
- // serving a cache-extended resource the url will change if the resource
- // does and so we've given it a long lifetime. If the site owner has done
- // something like set all css files to a 10-minute cache lifetime, that
- // shouldn't apply to our generated resources. See Apache code in
- // net/instaweb/apache/header_util:AddResponseHeadersToRequest
-
- // Make copies of name and value to put into headers_out.
- if (STR_EQ_LITERAL(name, "Cache-Control")) {
- ps_set_cache_control(r, reinterpret_cast(value.data));
- continue;
- } else if (STR_EQ_LITERAL(name, "Content-Type")) {
- // Unlike all the other headers, content_type is just a string.
- headers_out->content_type = value;
-
- // We should not include the charset when determining content_type_len, so
- // scan for the ';' that marks the start of the charset part.
- for (ngx_uint_t i = 0; i < value.len; i++) {
- if (value.data[i] == ';') {
- break;
- }
- headers_out->content_type_len = i + 1;
- }
-
- // In ngx_http_test_content_type() nginx will allocate and calculate
- // content_type_lowcase if we leave it as null.
- headers_out->content_type_lowcase = NULL;
- continue;
- // TODO(oschaaf): are there any other headers we should not try to
- // copy here?
- } else if (STR_EQ_LITERAL(name, "Connection")) {
- continue;
- } else if (STR_EQ_LITERAL(name, "Keep-Alive")) {
- continue;
- } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
- continue;
- } else if (STR_EQ_LITERAL(name, "Vary") && value.len
- && STR_EQ_LITERAL(value, "Accept-Encoding")) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- ctx->psol_vary_accept_only = true;
- }
-
- ngx_table_elt_t* header = static_cast(
- ngx_list_push(&headers_out->headers));
- if (header == NULL) {
- return NGX_ERROR;
- }
-
- header->hash = 1; // Include this header in the output.
- header->key.data = name.data;
- header->key.len = name.len;
- header->value.data = value.data;
- header->value.len = value.len;
-
- // Populate the shortcuts to commonly used headers.
- if (STR_EQ_LITERAL(name, "Date")) {
- headers_out->date = header;
- } else if (STR_EQ_LITERAL(name, "Etag")) {
- headers_out->etag = header;
- } else if (STR_EQ_LITERAL(name, "Expires")) {
- headers_out->expires = header;
- } else if (STR_EQ_LITERAL(name, "Last-Modified")) {
- headers_out->last_modified = header;
- } else if (STR_EQ_LITERAL(name, "Location")) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- if (ctx->location_field_set) {
- headers_out->location = header;
- }
- } else if (STR_EQ_LITERAL(name, "Server")) {
- headers_out->server = header;
- } else if (STR_EQ_LITERAL(name, "Content-Length")) {
- int64 len;
- CHECK(pagespeed_headers.FindContentLength(&len));
- headers_out->content_length_n = len;
- headers_out->content_length = header;
- } else if (STR_EQ_LITERAL(name, "Content-Encoding")) {
- headers_out->content_encoding = header;
- } else if (STR_EQ_LITERAL(name, "Refresh")) {
- headers_out->refresh = header;
- } else if (STR_EQ_LITERAL(name, "Content-Range")) {
- headers_out->content_range = header;
- } else if (STR_EQ_LITERAL(name, "Accept-Ranges")) {
- headers_out->accept_ranges = header;
- } else if (STR_EQ_LITERAL(name, "WWW-Authenticate")) {
- headers_out->www_authenticate = header;
- }
- }
-
- return NGX_OK;
-}
-
-namespace {
-
-typedef struct {
- NgxRewriteDriverFactory* driver_factory;
- MessageHandler* handler;
-} ps_main_conf_t;
-
-typedef struct {
- // If pagespeed is configured in some server block but not this one our
- // per-request code will be invoked but server context will be null. In those
- // cases we need to short circuit, not changing anything. Currently our
- // header filter, body filter, and content handler all do this, but if anyone
- // adds another way for nginx to give us a request to process we need to check
- // there as well.
- NgxServerContext* server_context;
- ProxyFetchFactory* proxy_fetch_factory;
- // Only used while parsing config. After we merge cfg_s and cfg_m you most
- // likely want cfg_s->server_context->config() as options here will be NULL.
- NgxRewriteOptions* options;
- MessageHandler* handler;
-} ps_srv_conf_t;
-
-typedef struct {
- NgxRewriteOptions* options;
- MessageHandler* handler;
-} ps_loc_conf_t;
-
-namespace RequestRouting {
-enum Response {
- kError,
- kStaticContent,
- kInvalidUrl,
- kPagespeedDisabled,
- kBeacon,
- kStatistics,
- kGlobalStatistics,
- kConsole,
- kMessages,
- kAdmin,
- kCachePurge,
- kGlobalAdmin,
- kPagespeedSubrequest,
- kErrorResponse,
- kResource,
-};
-} // namespace RequestRouting
-
-char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
-char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
-char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
-
-// We want NGX_CONF_MULTI for some very old versions:
-// https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
-// but it's gone in recent revisions, so provide a compat #define if needed
-#ifndef NGX_CONF_MULTI
-#define NGX_CONF_MULTI 0
-#endif
-
-// TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
-// not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
-ngx_command_t ps_commands[] = {
- { ngx_string("pagespeed"),
- NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
- NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
- ps_main_configure,
- NGX_HTTP_SRV_CONF_OFFSET,
- 0,
- NULL },
- { ngx_string("pagespeed"),
- NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
- NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
- ps_srv_configure,
- NGX_HTTP_SRV_CONF_OFFSET,
- 0,
- NULL },
-
- { ngx_string("pagespeed"),
- NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
- NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
- ps_loc_configure,
- NGX_HTTP_SRV_CONF_OFFSET,
- 0,
- NULL },
-
- ngx_null_command
-};
-
-bool ps_disabled(ps_srv_conf_t* cfg_s) {
- return cfg_s->server_context == NULL ||
- cfg_s->server_context->config()->unplugged();
-}
-
-void ps_ignore_sigpipe() {
- struct sigaction act;
- ngx_memzero(&act, sizeof(act));
- act.sa_handler = SIG_IGN;
- sigemptyset(&act.sa_mask);
- act.sa_flags = 0;
- sigaction(SIGPIPE, &act, NULL);
-}
-
-// Given a directory path that pagespeed needs, create it and set permissions so
-// the worker can access, but only if needed.
-char* ps_init_dir(const StringPiece& directive,
- const StringPiece& path,
- ngx_conf_t* cf) {
- if (path.size() == 0 || path[0] != '/') {
- return string_piece_to_pool_string(
- cf->pool, net_instaweb::StrCat(directive, " ", path,
- " must start with a slash"));
- }
-
- net_instaweb::StdioFileSystem file_system;
- net_instaweb::NullMessageHandler message_handler;
- GoogleString gs_path;
- path.CopyToString(&gs_path);
- if (!file_system.IsDir(gs_path.c_str(), &message_handler).is_true()) {
- if (!file_system.RecursivelyMakeDir(path, &message_handler)) {
- return string_piece_to_pool_string(
- cf->pool, net_instaweb::StrCat(
- directive, " path ", path,
- " does not exist and could not be created."));
- }
- // Directory created, but may not be readable by the worker processes.
- }
-
- if (geteuid() != 0) {
- return NULL; // We're not root, so we're staying whoever we are.
- }
-
- // chown if owner differs from nginx worker user.
- ngx_core_conf_t* ccf = reinterpret_cast(
- ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
- CHECK(ccf != NULL);
- struct stat gs_stat;
- if (stat(gs_path.c_str(), &gs_stat) != 0) {
- return string_piece_to_pool_string(
- cf->pool, net_instaweb::StrCat(
- directive, " ", path, " stat() failed"));
- }
- if (gs_stat.st_uid != ccf->user) {
- if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
- return string_piece_to_pool_string(
- cf->pool, net_instaweb::StrCat(
- directive, " ", path, " unable to set permissions"));
- }
- }
-
- return NULL;
-}
-
-// We support interpretation of nginx variables in some configuration settings,
-// but we also need to support literal dollar signs in those same settings.
-// Nginx has no good solution for this, so we define $dollar to expand to '$',
-// which lets people include literal dollar signs if they need them.
-ngx_int_t ps_dollar(
- ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) {
- v->valid = 1;
- v->no_cacheable = 0;
- v->not_found = 0;
- v->data = reinterpret_cast(const_cast("$"));
- v->len = 1;
- return NGX_OK;
-}
-
-// Parse the configuration option represented by cf and add it to options,
-// creating options if necessary.
-char* ps_configure(ngx_conf_t* cf,
- NgxRewriteOptions** options,
- MessageHandler* handler,
- net_instaweb::RewriteOptions::OptionScope option_scope) {
- // args[0] is always "pagespeed"; ignore it.
- ngx_uint_t n_args = cf->args->nelts - 1;
-
- // In ps_commands we only register 'pagespeed' as taking up to
- // five arguments, so this check should never fire.
- CHECK(n_args <= NGX_PAGESPEED_MAX_ARGS);
- StringPiece args[NGX_PAGESPEED_MAX_ARGS];
-
- ngx_str_t* value = static_cast(cf->args->elts);
- ngx_uint_t i;
- for (i = 0 ; i < n_args ; i++) {
- args[i] = str_to_string_piece(value[i+1]);
- }
-
- if (n_args == 1) {
- if (StringCaseEqual(args[0], "on")) {
- // safe to call if the setter is disabled
- g_gzip_setter.EnableGZipForLocation(cf);
- } else if (StringCaseEqual(args[0], "off")) {
- g_gzip_setter.SetGZipForLocation(cf, false);
- }
- }
- if (n_args == 2 && args[0].compare("gzip") == 0) {
- if (args[1].compare("on") == 0) {
- g_gzip_setter.SetGZipForLocation(cf, true);
- } else if (args[1].compare("off") == 0) {
- g_gzip_setter.SetGZipForLocation(cf, false);
- } else {
- char* error_message = string_piece_to_pool_string(
- cf->pool, StringPiece("Invalid pagespeed gzip setting"));
- return error_message;
- }
- return NGX_CONF_OK;
- }
-
- // Some options require the worker process to be able to read and write to
- // a specific directory. Generally the master process is root while the
- // worker is nobody, so we need to change permissions and create the directory
- // if necessary.
- if (n_args == 2 &&
- (net_instaweb::StringCaseEqual("LogDir", args[0]) ||
- net_instaweb::StringCaseEqual("FileCachePath", args[0]))) {
- char* error_message = ps_init_dir(args[0], args[1], cf);
- if (error_message != NULL) {
- return error_message;
- }
- // The directory has been prepared, but we haven't actually parsed the
- // directive yet. That happens below in ParseAndSetOptions().
- }
-
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
- if (*options == NULL) {
- *options = new NgxRewriteOptions(cfg_m->driver_factory->thread_system());
- }
-
- ProcessScriptVariablesMode script_mode =
- dynamic_cast(cfg_m->driver_factory)
- ->process_script_variables();
- if (script_mode != ProcessScriptVariablesMode::kOff) {
- // To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
- ngx_str_t name = ngx_string("ps_dollar");
- ngx_http_variable_t* var =
- ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
-
- if (var == NULL) {
- return const_cast(
- "Failed to add global configuration variable for '$ps_dollar'");
- }
- var->get_handler = ps_dollar;
- }
-
- const char* status = (*options)->ParseAndSetOptions(
- args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
- script_mode);
-
- // nginx expects us to return a string literal but doesn't mark it const.
- return const_cast(status);
-}
-
-char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- ps_srv_conf_t* cfg_s = static_cast(
- ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
- return ps_configure(cf, &cfg_s->options, cfg_s->handler,
- net_instaweb::RewriteOptions::kProcessScopeStrict);
-}
-
-char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- ps_srv_conf_t* cfg_s = static_cast(
- ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
- return ps_configure(cf, &cfg_s->options, cfg_s->handler,
- net_instaweb::RewriteOptions::kServerScope);
-}
-
-char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
- ps_loc_conf_t* cfg_l = static_cast(
- ngx_http_conf_get_module_loc_conf(cf, ngx_pagespeed));
- return ps_configure(cf, &cfg_l->options, cfg_l->handler,
- net_instaweb::RewriteOptions::kDirectoryScope);
-}
-
-void ps_cleanup_loc_conf(void* data) {
- ps_loc_conf_t* cfg_l = static_cast(data);
- delete cfg_l->handler;
- cfg_l->handler = NULL;
- delete cfg_l->options;
- cfg_l->options = NULL;
-}
-
-bool factory_deleted = false;
-void ps_cleanup_srv_conf(void* data) {
- ps_srv_conf_t* cfg_s = static_cast(data);
-
- // destroy the factory on the first call, causing all worker threads
- // to be shut down when we destroy any proxy_fetch_factories. This
- // will prevent any queued callbacks to destroyed proxy fetch factories
- // from being executed
- if (!factory_deleted && cfg_s->server_context != NULL) {
- if (active_driver_factory == cfg_s->server_context->factory()) {
- active_driver_factory = NULL;
- }
- delete cfg_s->server_context->factory();
- factory_deleted = true;
- }
- if (cfg_s->proxy_fetch_factory != NULL) {
- delete cfg_s->proxy_fetch_factory;
- cfg_s->proxy_fetch_factory = NULL;
- }
- delete cfg_s->handler;
- cfg_s->handler = NULL;
- delete cfg_s->options;
- cfg_s->options = NULL;
-}
-
-void ps_cleanup_main_conf(void* data) {
- ps_main_conf_t* cfg_m = static_cast(data);
- delete cfg_m->handler;
- cfg_m->handler = NULL;
- NgxRewriteDriverFactory::Terminate();
- NgxRewriteOptions::Terminate();
-
- // reset the factory deleted flag, so we will clean up properly next time,
- // in case of a configuration reload.
- // TODO(oschaaf): get rid of the factory_deleted flag
- factory_deleted = false;
-}
-
-template ConfT* ps_create_conf(ngx_conf_t* cf) {
- ConfT* cfg = static_cast(ngx_pcalloc(cf->pool, sizeof(ConfT)));
- if (cfg == NULL) {
- return NULL;
- }
- cfg->handler = new GoogleMessageHandler();
- return cfg;
-}
-
-void ps_set_conf_cleanup_handler(
- ngx_conf_t* cf, void (func)(void*), void* data) { // NOLINT
- ngx_pool_cleanup_t* cleanup_m = ngx_pool_cleanup_add(cf->pool, 0);
- if (cleanup_m == NULL) {
- ngx_conf_log_error(
- NGX_LOG_ERR, cf, 0, "failed to register a cleanup handler");
- } else {
- cleanup_m->handler = func;
- cleanup_m->data = data;
- }
-}
-
-void terminate_process_context() {
- if (active_driver_factory != NULL) {
- // If we got here, that means we are in the cache loader/manager
- // or did not get a chance to cleanup otherwise.
- delete active_driver_factory;
- active_driver_factory = NULL;
- NgxBaseFetch::Terminate();
- }
- delete process_context;
- process_context = NULL;
-}
-
-void* ps_create_main_conf(ngx_conf_t* cf) {
- if (!process_context_cleanup_hooked) {
- SystemRewriteDriverFactory::InitApr();
- atexit(terminate_process_context);
- process_context_cleanup_hooked = true;
- }
- ps_main_conf_t* cfg_m = ps_create_conf(cf);
- if (cfg_m == NULL) {
- return NGX_CONF_ERROR;
- }
- CHECK(!factory_deleted);
- NgxRewriteOptions::Initialize();
- NgxRewriteDriverFactory::Initialize();
-
- cfg_m->driver_factory = new NgxRewriteDriverFactory(
- *process_context,
- new SystemThreadSystem(),
- "" /* hostname, not used */,
- -1 /* port, not used */);
- active_driver_factory = cfg_m->driver_factory;
- active_driver_factory->LoggingInit(ngx_cycle->log, false);
- cfg_m->driver_factory->Init();
- ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
- return cfg_m;
-}
-
-void* ps_create_srv_conf(ngx_conf_t* cf) {
- ps_srv_conf_t* cfg_s = ps_create_conf(cf);
- if (cfg_s == NULL) {
- return NGX_CONF_ERROR;
- }
- ps_set_conf_cleanup_handler(cf, ps_cleanup_srv_conf, cfg_s);
- return cfg_s;
-}
-
-void* ps_create_loc_conf(ngx_conf_t* cf) {
- ps_loc_conf_t* cfg_l = ps_create_conf(cf);
- if (cfg_l == NULL) {
- return NGX_CONF_ERROR;
- }
- ps_set_conf_cleanup_handler(cf, ps_cleanup_loc_conf, cfg_l);
- return cfg_l;
-}
-
-// nginx has hierarchical configuration. It maintains configurations at many
-// levels. At various points it needs to merge configurations from different
-// levels, and then it calls this. First it creates the configuration at the
-// new level, parsing any pagespeed directives, then it merges in the
-// configuration from the level above. This function should merge the parent
-// configuration into the child. It's more complex than options->Merge() both
-// because of the cases where the parent or child didn't have any pagespeed
-// directives and because merging is order-dependent in the opposite way we'd
-// like.
-void ps_merge_options(NgxRewriteOptions* parent_options,
- NgxRewriteOptions** child_options) {
- if (parent_options == NULL) {
- // Nothing to do.
- } else if (*child_options == NULL) {
- *child_options = parent_options->Clone();
- } else { // Both non-null.
- // Unfortunately, merging configuration options is order dependent. We'd
- // like to just do (*child_options)->Merge(*parent_options)
- // but then if we had:
- // pagespeed RewriteLevel PassThrough
- // server {
- // pagespeed RewriteLevel CoreFilters
- // }
- // it would always be stuck on PassThrough.
- NgxRewriteOptions* child_specific_options = *child_options;
- *child_options = parent_options->Clone();
- (*child_options)->Merge(*child_specific_options);
-
- if (child_specific_options->clear_inherited_scripts()) {
- // We don't want to inherit any inherited script lines from the parent
- // options here, so we just stick to the child specific ones.
- child_specific_options->CopyScriptLinesTo(*child_options);
- } else {
- // We append the child specific script lines to the parent's script lines
- // so we preserve the order in which they will be executed at request time
- child_specific_options->AppendScriptLinesTo(*child_options);
- }
- delete child_specific_options;
- }
-}
-
-namespace {
-
-int times_ps_merge_srv_conf_called = 0;
-
-} // namespace
-
-// Called exactly once per server block to merge the main configuration with the
-// configuration for this server.
-char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
- times_ps_merge_srv_conf_called += 1;
-
- ps_srv_conf_t* parent_cfg_s = static_cast(parent);
- ps_srv_conf_t* cfg_s = static_cast(child);
-
- ps_merge_options(parent_cfg_s->options, &cfg_s->options);
-
- if (cfg_s->options == NULL) {
- return NGX_CONF_OK; // No pagespeed options; don't do anything.
- }
-
- // ServerContext needs a hostname and port, but I don't see how to get this
- // and it ignores that a server can have multiple names and ports. Because
- // the server context only needs them to make a unique identifier and to make
- // debugging easier, substitute our own unique identifier.
- // TODO(jefftk): either figure out how to get a hostname and port for this
- // server block or change ServerContext not to ask for them.
- int dummy_port = -times_ps_merge_srv_conf_called;
-
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
- cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
- cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
- "dummy_hostname", dummy_port);
-
-#if (NGX_HTTP_V2)
- // Save the variable index of the "http2" variable, so we can use it
- // at request time to lookup whether that's on. We do this conditionally
- // since NGINX will complain to the user (at [emerg] level!) if it doesn't
- // know about it.
- ngx_str_t name = ngx_string("http2");
- cfg_s->server_context->set_ngx_http2_variable_index(
- ngx_http_get_variable_index(cf, &name));
-#endif
-
- // The server context sets some options when we call global_options(). So
- // let it do that, then merge in options we got from the config file.
- // Once we do that we're done with cfg_s->options.
- cfg_s->server_context->global_options()->Merge(*cfg_s->options);
- NgxRewriteOptions* ngx_options = dynamic_cast(
- cfg_s->server_context->global_options());
- cfg_s->options->CopyScriptLinesTo(ngx_options);
- delete cfg_s->options;
- cfg_s->options = NULL;
-
- if (!cfg_s->server_context->global_options()->unplugged()) {
- // Validate FileCachePath
- GoogleMessageHandler handler;
- const char* file_cache_path =
- cfg_s->server_context->config()->file_cache_path().c_str();
- if (file_cache_path[0] == '\0') {
- if (!cfg_s->server_context->global_options()->standby()) {
- return const_cast("FileCachePath must be set, even for standby");
- } else {
- return const_cast("FileCachePath must be set");
- }
- } else if (!cfg_m->driver_factory->file_system()->IsDir(
- file_cache_path, &handler).is_true()) {
- return const_cast(
- "FileCachePath must be an nginx-writeable directory");
- }
- }
-
- return NGX_CONF_OK;
-}
-
-char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
- ps_loc_conf_t* cfg_l = static_cast(child);
- if (cfg_l->options == NULL) {
- // No directory specific options.
- return NGX_CONF_OK;
- }
-
- // While you can't put a "location" block inside a "location" block you can
- // put an "if" block inside a "location" block, which is implemented by making
- // a pretend "location" block. In this case we may have pagespeed options
- // from the parent "location" block as well as from the current locationish
- // "if" block.
- ps_loc_conf_t* parent_cfg_l = static_cast(parent);
- if (parent_cfg_l->options != NULL) {
- // Rebase our options off of the ones defined in the parent location block.
- ps_merge_options(parent_cfg_l->options, &cfg_l->options);
- return NGX_CONF_OK;
- }
-
- // Pagespeed options are defined in this location block, and it either has no
- // parent (typical case) or is an if block whose parent location block defines
- // no pagespeed options. Base our options off of those in the server block.
-
- ps_srv_conf_t* cfg_s = static_cast(
- ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
-
- if (ps_disabled(cfg_s)) {
- // Pagespeed options cannot be defined only in location blocks. There must
- // be at least a single "pagespeed off" in the main block or a server
- // block.
- return NGX_CONF_OK;
- }
-
- // If we get here we have parent options ("global options") from cfg_s, child
- // options ("directory specific options") from cfg_l, and no options from
- // parent_cfg_l. Rebase the directory specific options on the global options.
- ps_merge_options(cfg_s->server_context->config(), &cfg_l->options);
-
- return NGX_CONF_OK;
-}
-
-// _ef_ is a shorthand for ETag Filter
-ngx_http_output_header_filter_pt ngx_http_ef_next_header_filter;
-
-bool ps_is_https(ngx_http_request_t* r) {
- // Based on ngx_http_variable_scheme.
-#if (NGX_HTTP_SSL)
- return r->connection->ssl;
-#endif
- return false;
-}
-
-int ps_determine_port(ngx_http_request_t* r) {
- // Return -1 if the port isn't specified, the port number otherwise.
- //
- // If a Host header was provided, get the host from that. Otherwise fall back
- // to the local port of the incoming connection.
-
- int port = -1;
- ngx_table_elt_t* host = r->headers_in.host;
-
- if (host != NULL) {
- // Host headers can look like:
- //
- // www.example.com // normal
- // www.example.com:8080 // port specified
- // 127.0.0.1 // IPv4
- // 127.0.0.1:8080 // IPv4 with port
- // [::1] // IPv6
- // [::1]:8080 // IPv6 with port
- //
- // The IPv6 ones are the annoying ones, but the square brackets allow us to
- // disambiguate. To find the port number, we can say:
- //
- // 1) Take the text after the final colon.
- // 2) If all of those characters are digits, that's your port number
- //
- // In the case of a plain IPv6 address with no port number, the text after
- // the final colon will include a ']', so we'll stop processing.
-
- StringPiece host_str = str_to_string_piece(host->value);
- size_t colon_index = host_str.rfind(":");
- if (colon_index == host_str.npos) {
- return -1;
- }
- // Strip everything up to and including the final colon.
- host_str.remove_prefix(colon_index + 1);
-
- bool ok = StringToInt(host_str, &port);
- if (!ok) {
- // Might be malformed port, or just IPv6 with no port specified.
- return -1;
- }
-
- return port;
- }
-
- // Based on ngx_http_variable_server_port.
-#if (NGX_HAVE_INET6)
- if (r->connection->local_sockaddr->sa_family == AF_INET6) {
- port = ntohs(reinterpret_cast(
- r->connection->local_sockaddr)->sin6_port);
- }
-#endif
- if (port == -1 /* still need port */) {
- port = ntohs(reinterpret_cast(
- r->connection->local_sockaddr)->sin_port);
- }
-
- return port;
-}
-} // namespace
-
-StringPiece ps_determine_host(ngx_http_request_t* r) {
- StringPiece host = str_to_string_piece(r->headers_in.server);
- if (host.size() == 0) {
- // If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
- // fall back to server IP address. Based on ngx_http_variable_server_addr.
- ngx_str_t s;
- u_char addr[NGX_SOCKADDR_STRLEN];
- s.len = NGX_SOCKADDR_STRLEN;
- s.data = addr;
- ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &s, 0);
- if (rc != NGX_OK) {
- s.len = 0;
- }
- host = str_to_string_piece(s);
- }
- return host;
-}
-
-namespace {
-
-GoogleString ps_determine_url(ngx_http_request_t* r) {
- int port = ps_determine_port(r);
- GoogleString port_string;
- if ((ps_is_https(r) && (port == 443 || port == -1)) ||
- (!ps_is_https(r) && (port == 80 || port == -1))) {
- // No port specifier needed for requests on default ports.
- port_string = "";
- } else {
- port_string = StrCat(":", IntegerToString(port));
- }
-
- StringPiece host = ps_determine_host(r);
-
- return StrCat(ps_is_https(r) ? "https://" : "http://",
- host, port_string, str_to_string_piece(r->unparsed_uri));
-}
-
-// we are still at pagespeed phase
-ngx_int_t ps_decline_request(ngx_http_request_t* r) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- CHECK(ctx != NULL);
-
- ctx->driver->Cleanup();
- ctx->driver = NULL;
- ctx->location_field_set = false;
- ctx->psol_vary_accept_only = false;
-
- // re init ctx
- ctx->html_rewrite = true;
- ctx->in_place = false;
- ps_release_base_fetch(ctx);
- ps_set_buffered(r, false);
-
- r->count++;
- r->phase_handler++;
-
- //restore read_event_handler to what it was in ps_async_wait_response
- r->read_event_handler = ngx_http_block_reading;
- r->write_event_handler = ngx_http_core_run_phases;
- ngx_http_core_run_phases(r);
- ngx_http_run_posted_requests(r->connection);
- return NGX_DONE;
-}
-
-ngx_int_t ps_async_wait_response(ngx_http_request_t* r) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- CHECK(ctx != NULL);
-
- r->count++;
- // While we wait for PSOL to complete an async operation, there is a chance
- // that the underlying connection gets closed, or a http/2 RST_STREAM is
- // received before the async operation completes. In that case we don't want
- // to continue processing this flow. So we override the requests's read event
- // handler with one that will make nginx abort request processing and execute
- // our cleanup handlers instead of resuming request processing.
- r->read_event_handler = ngx_http_test_reading;
- r->write_event_handler = ngx_http_request_empty_handler;
- ps_set_buffered(r, true);
- // We don't need to add a timer here, as it will be set by nginx.
- return NGX_DONE;
-}
-
-// Populate cfg_* with configuration information for this request.
-// Thin wrappers around ngx_http_get_module_*_conf and cast.
-ps_srv_conf_t* ps_get_srv_config(ngx_http_request_t* r) {
- return static_cast(
- ngx_http_get_module_srv_conf(r, ngx_pagespeed));
-}
-ps_loc_conf_t* ps_get_loc_config(ngx_http_request_t* r) {
- return static_cast(
- ngx_http_get_module_loc_conf(r, ngx_pagespeed));
-}
-
-RewriteOptions* ps_determine_remote_options(ps_srv_conf_t* cfg_s) {
- if (!cfg_s || !cfg_s->server_context ||
- !cfg_s->server_context->global_options()) {
- return NULL;
- }
- if (!cfg_s->server_context->global_options()
- ->remote_configuration_url()
- .empty()) {
- RewriteOptions* remote_options =
- cfg_s->server_context->global_options()->Clone();
- // This fetch is blocking for up to remote_configuration_timeout_ms ms.
- cfg_s->server_context->GetRemoteOptions(remote_options, false);
- return remote_options;
- }
- return NULL;
-}
-
-// Wrapper around GetQueryOptions()
-RewriteOptions* ps_determine_request_options(
- ngx_http_request_t* r,
- const RewriteOptions* domain_options, /* may be null */
- RequestHeaders* request_headers,
- ResponseHeaders* response_headers,
- RequestContextPtr request_context,
- ps_srv_conf_t* cfg_s,
- GoogleUrl* url,
- GoogleString* pagespeed_query_params,
- GoogleString* pagespeed_option_cookies) {
- // Sets option from request headers and url.
- RewriteQuery rewrite_query;
- if (!cfg_s->server_context->GetQueryOptions(
- request_context, domain_options, url, request_headers,
- response_headers, &rewrite_query)) {
- // Failed to parse query params or request headers. Treat this as if there
- // were no query params given.
- ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
- "ps_determine_request_options: parsing headers or query params failed.");
- return NULL;
- }
-
- *pagespeed_query_params =
- rewrite_query.pagespeed_query_params().ToEscapedString();
- *pagespeed_option_cookies =
- rewrite_query.pagespeed_option_cookies().ToEscapedString();
-
- // Will be NULL if there aren't any options set with query params or in
- // headers.
- return rewrite_query.ReleaseOptions();
-}
-
-// Check whether this visitor is already in an experiment. If they're not,
-// classify them into one by setting a cookie. Then set options appropriately
-// for their experiment.
-//
-// See InstawebContext::SetExperimentStateAndCookie()
-bool ps_set_experiment_state_and_cookie(ngx_http_request_t* r,
- RequestHeaders* request_headers,
- RewriteOptions* options,
- const StringPiece& host) {
- CHECK(options->running_experiment());
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- bool need_cookie =
- cfg_s->server_context->experiment_matcher()->ClassifyIntoExperiment(
- *request_headers, *cfg_s->server_context->user_agent_matcher(),
- options);
- if (need_cookie && host.length() > 0) {
- PosixTimer timer;
- int64 time_now_ms = timer.NowMs();
- int64 expiration_time_ms = (time_now_ms +
- options->experiment_cookie_duration_ms());
-
- // TODO(jefftk): refactor SetExperimentCookie to expose the value we want to
- // set on the cookie.
- int state = options->experiment_id();
- GoogleString expires;
- ConvertTimeToString(expiration_time_ms, &expires);
- GoogleString value = StringPrintf(
- "%s=%s; Expires=%s; Domain=.%s; Path=/",
- experiment::kExperimentCookie,
- experiment::ExperimentStateToCookieString(state).c_str(),
- expires.c_str(), host.as_string().c_str());
-
- // Set the PagespeedExperiment cookie.
- ngx_table_elt_t* cookie = static_cast(
- ngx_list_push(&r->headers_out.headers));
- if (cookie == NULL) {
- return false;
- }
- cookie->hash = 1; // Include this header in the response.
-
- ngx_str_set(&cookie->key, "Set-Cookie");
- // It's not safe to use value.c_str here because cookie header only keeps a
- // pointer to the string data.
- cookie->value.data = reinterpret_cast(
- string_piece_to_pool_string(r->pool, value));
- cookie->value.len = value.size();
- }
- return true;
-}
-
-// There are many sources of options:
-// - the request (query parameters, headers, and cookies)
-// - location block
-// - global server options
-// - experiment framework
-// Consider them all, returning appropriate options for this request, of which
-// the caller takes ownership. If the only applicable options are global,
-// set options to NULL so we can use server_context->global_options().
-bool ps_determine_options(ngx_http_request_t* r,
- RequestHeaders* request_headers,
- ResponseHeaders* response_headers,
- RewriteOptions** options,
- RequestContextPtr request_context,
- ps_srv_conf_t* cfg_s,
- GoogleUrl* url,
- GoogleString* pagespeed_query_params,
- GoogleString* pagespeed_option_cookies,
- bool html_rewrite) {
- ps_loc_conf_t* cfg_l = ps_get_loc_config(r);
-
- // Global options for this server. Never null.
- RewriteOptions* global_options = cfg_s->server_context->global_options();
-
- // Directory-specific options, usually null. They've already been rebased off
- // of the global options as part of the configuration process.
- RewriteOptions* directory_options = cfg_l->options;
-
- // Request-specific options, nearly always null. If set they need to be
- // rebased on the directory options or the global options.
- RewriteOptions* request_options = ps_determine_request_options(
- r, directory_options, request_headers, response_headers, request_context,
- cfg_s, url, pagespeed_query_params, pagespeed_option_cookies);
- bool have_request_options = request_options != NULL;
-
- // Because the caller takes ownership of any options we return, the only
- // situation in which we can avoid allocating a new RewriteOptions is if the
- // global options are ok as they are and we don't have script variables we
- // need to evaluate at this point.
- NgxRewriteOptions* ngx_global_options =
- dynamic_cast(global_options);
- if (!have_request_options && directory_options == NULL &&
- !global_options->running_experiment() &&
- ngx_global_options->script_lines().size() == 0) {
- return true;
- }
-
- // Start with directory options if we have them, otherwise request options.
- if (directory_options != NULL) {
- if (*options != NULL) {
- (*options)->Merge(*directory_options);
- } else {
- *options = directory_options->Clone();
- }
- } else {
- if (*options == NULL) {
- *options = global_options->Clone();
- }
- }
-
- NgxRewriteDriverFactory* ngx_factory =
- dynamic_cast(cfg_s->server_context->factory());
- NgxRewriteOptions* ngx_options = dynamic_cast(*options);
-
- // ExecuteScriptVariables() sets 'pagespeed off' on ngx_options when execution
- // fails and then returns false. When that happens we return, as we don't want
- // to allow enabling pagespeed by request and execute without the intended
- // configuration.
- if (!ngx_options->ExecuteScriptVariables(r, cfg_s->handler, ngx_factory)) {
- return false;
- }
-
- // Modify our options in response to request options if specified.
- if (have_request_options) {
- (*options)->Merge(*request_options);
- delete request_options;
- request_options = NULL;
- }
-
- // If we're running an experiment and processing html then modify our options
- // in response to the experiment. Except we generally don't want experiments
- // to be contaminated with unexpected settings, so ignore experiments if we
- // have request-specific options. Unless EnrollExperiment is on, probably set
- // by a query parameter, in which case we want to go ahead and apply the
- // experimental settings even if it means bad data, because we're just seeing
- // what it looks like.
- if ((*options)->running_experiment() &&
- html_rewrite &&
- (!have_request_options ||
- (*options)->enroll_experiment())) {
- bool ok = ps_set_experiment_state_and_cookie(
- r, request_headers, *options, url->Host());
- if (!ok) {
- delete *options;
- *options = NULL;
- return false;
- }
- }
-
- return true;
-}
-
-// Fix URL based on X-Forwarded-Proto.
-// http://code.google.com/p/modpagespeed/issues/detail?id=546 For example, if
-// Apache gives us the URL "http://www.example.com/" and there is a header:
-// "X-Forwarded-Proto: https", then we update this base URL to
-// "https://www.example.com/". This only ever changes the protocol of the url.
-//
-// Returns true if it modified url, false otherwise.
-bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
- // First check for an X-Forwarded-Proto header.
- const ngx_str_t* x_forwarded_proto_header = NULL;
-
- ngx_table_elt_t* header;
- NgxListIterator it(&(r->headers_in.headers.part));
- while ((header = it.Next()) != NULL) {
- if (STR_CASE_EQ_LITERAL(header->key, "X-Forwarded-Proto")) {
- x_forwarded_proto_header = &header->value;
- break;
- }
- }
-
- if (x_forwarded_proto_header == NULL) {
- return false; // No X-Forwarded-Proto header found.
- }
-
- StringPiece x_forwarded_proto =
- str_to_string_piece(*x_forwarded_proto_header);
- if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
- !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
- LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
- << " for URL " << url << " protocol not changed.";
- return false;
- }
-
- StringPiece url_sp(*url);
- StringPiece::size_type colon_pos = url_sp.find(":");
-
- if (colon_pos == StringPiece::npos) {
- return false; // URL appears to have no protocol; give up.
- }
-
- // Replace URL protocol with that specified in X-Forwarded-Proto.
- *url = StrCat(x_forwarded_proto, url_sp.substr(colon_pos));
-
- return true;
-}
-
-bool is_pagespeed_subrequest(ngx_http_request_t* r) {
- ngx_table_elt_t* user_agent_header = r->headers_in.user_agent;
- if (user_agent_header == NULL) {
- return false;
- }
- StringPiece user_agent = str_to_string_piece(user_agent_header->value);
- return (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos);
-}
-
-void ps_release_base_fetch(ps_request_ctx_t* ctx) {
- // In the normal flow BaseFetch doesn't delete itself in HandleDone() because
- // we still need to receive notification via pipe and call
- // CollectAccumulatedWrites. If there's an error and we're cleaning up early
- // then HandleDone() hasn't been called yet and we need the base fetch to wait
- // for that and then delete itself.
- if (ctx->base_fetch != NULL) {
- ctx->base_fetch->Detach();
- ctx->base_fetch = NULL;
- }
-}
-
-// TODO(chaizhenhua): merge into NgxBaseFetch ctor
-void ps_create_base_fetch(StringPiece url,
- ps_request_ctx_t* ctx,
- RequestContextPtr request_context,
- RequestHeaders* request_headers,
- NgxBaseFetchType type,
- const RewriteOptions* options) {
- CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
-
- ngx_http_request_t* r = ctx->r;
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-
- // Handles its own deletion. We need to call Release when we're done with
- // it, and call Done() on the associated parent (Proxy or Resource) fetch. If
- // we fail before creating the associated fetch then we need to call Done() on
- // the BaseFetch ourselves.
- ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
- ctx->preserve_caching_headers, type,
- options);
- ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
-}
-
-void ps_release_request_context(void* data) {
- ps_request_ctx_t* ctx = static_cast(data);
-
- // proxy_fetch deleted itself if we called Done(), but if an error happened
- // before then we need to tell it to delete itself.
- //
- // If this is a resource fetch then proxy_fetch was never initialized.
- if (ctx->proxy_fetch != NULL) {
- ctx->proxy_fetch->Done(false /* failure */);
- ctx->proxy_fetch = NULL;
- }
-
- if (ctx->inflater_ != NULL) {
- delete ctx->inflater_;
- ctx->inflater_ = NULL;
- }
-
- if (ctx->driver != NULL) {
- ctx->driver->Cleanup();
- ctx->driver = NULL;
- }
-
- if (ctx->recorder != NULL) {
- // Deletes recorder.
- ctx->recorder->DoneAndSetHeaders(NULL, false /* incomplete response */);
- ctx->recorder = NULL;
- }
-
- ps_release_base_fetch(ctx);
- delete ctx;
-}
-
-// Set us up for processing a request. Creates a request context and determines
-// which handler should deal with the request.
-RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
-
- if (ps_disabled(cfg_s)) {
- // Not enabled for this server block.
- return RequestRouting::kPagespeedDisabled;
- }
-
- if (ngx_terminate || ngx_exiting) {
- return RequestRouting::kError;
- }
- if (r->err_status != 0) {
- return RequestRouting::kErrorResponse;
- }
-
- GoogleString url_string = ps_determine_url(r);
- GoogleUrl url(url_string);
-
- if (!url.IsWebValid()) {
- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
-
- // Let nginx deal with the error however it wants; we will see a NULL ctx in
- // the body filter or content handler and do nothing.
- return RequestRouting::kInvalidUrl;
- }
-
- if (is_pagespeed_subrequest(r)) {
- return RequestRouting::kPagespeedSubrequest;
- } else if (
- url.PathSansLeaf() == dynamic_cast(
- cfg_s->server_context->factory())->static_asset_prefix()) {
- return RequestRouting::kStaticContent;
- }
-
- const NgxRewriteOptions* global_options = cfg_s->server_context->config();
-
- StringPiece path = url.PathSansQuery();
- if (StringCaseEqual(path, global_options->statistics_path()) &&
- global_options->StatisticsAccessAllowed(url)) {
- return RequestRouting::kStatistics;
- } else if (StringCaseEqual(path, global_options->global_statistics_path()) &&
- global_options->GlobalStatisticsAccessAllowed(url)) {
- return RequestRouting::kGlobalStatistics;
- } else if (StringCaseEqual(path, global_options->console_path()) &&
- global_options->ConsoleAccessAllowed(url)) {
- return RequestRouting::kConsole;
- } else if (StringCaseEqual(path, global_options->messages_path()) &&
- global_options->MessagesAccessAllowed(url)) {
- return RequestRouting::kMessages;
- } else if (
- // The admin handlers get everything under a path (/path/*) while all the
- // other handlers only get exact matches (/path). So match all paths
- // starting with the handler path.
- !global_options->admin_path().empty() &&
- StringCaseStartsWith(path, global_options->admin_path()) &&
- global_options->AdminAccessAllowed(url)) {
- return RequestRouting::kAdmin;
- } else if (!global_options->global_admin_path().empty() &&
- StringCaseStartsWith(path, global_options->global_admin_path()) &&
- global_options->GlobalAdminAccessAllowed(url)) {
- return RequestRouting::kGlobalAdmin;
- } else if (global_options->enable_cache_purge() &&
- !global_options->purge_method().empty() &&
- (global_options->purge_method() ==
- str_to_string_piece(r->method_name))) {
- return RequestRouting::kCachePurge;
- }
-
- const GoogleString* beacon_url;
- if (ps_is_https(r)) {
- beacon_url = &(global_options->beacon_url().https);
- } else {
- beacon_url = &(global_options->beacon_url().http);
- }
-
- if (url.PathSansQuery() == StringPiece(*beacon_url)) {
- return RequestRouting::kBeacon;
- }
-
- return RequestRouting::kResource;
-}
-
-ngx_int_t ps_resource_handler(ngx_http_request_t* r,
- bool html_rewrite,
- RequestRouting::Response response_category) {
- if (r != r->main) {
- return NGX_DECLINED;
- }
-
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- if (ngx_terminate || ngx_exiting) {
- cfg_s->server_context->message_handler()->Message(
- kInfo, "ps_resource_handler declining: nginx worker is shutting down");
-
- if (ctx == NULL) {
- return NGX_DECLINED;
- }
- ps_release_base_fetch(ctx);
- return NGX_DECLINED;
- }
-
- CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));
-
- if (!html_rewrite &&
- r->method != NGX_HTTP_GET &&
- r->method != NGX_HTTP_HEAD &&
- r->method != NGX_HTTP_POST &&
- response_category != RequestRouting::kCachePurge) {
- return NGX_DECLINED;
- }
-
- GoogleString url_string = ps_determine_url(r);
- GoogleUrl url(url_string);
-
- if (!url.IsWebValid()) {
- ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
- return NGX_DECLINED;
- }
-
- scoped_ptr request_headers(new RequestHeaders);
- scoped_ptr response_headers(new ResponseHeaders);
-
- copy_request_headers_from_ngx(r, request_headers.get());
- copy_response_headers_from_ngx(r, response_headers.get());
-
- RequestContextPtr request_context(
- cfg_s->server_context->NewRequestContext(r));
- GoogleString pagespeed_query_params;
- GoogleString pagespeed_option_cookies;
- RewriteOptions* options = ps_determine_remote_options(cfg_s);
- if (!ps_determine_options(r, request_headers.get(), response_headers.get(),
- &options, request_context, cfg_s, &url,
- &pagespeed_query_params, &pagespeed_option_cookies,
- html_rewrite)) {
- return NGX_ERROR;
- }
-
- // Take ownership of custom_options.
- scoped_ptr custom_options(options);
- if (options == NULL) {
- options = cfg_s->server_context->global_options();
- }
-
- request_context->set_options(options->ComputeHttpOptions());
-
- // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
- // parameters. Keep url_string in sync with url.
- url.Spec().CopyToString(&url_string);
-
- if (cfg_s->server_context->global_options()->respect_x_forwarded_proto()) {
- bool modified_url = ps_apply_x_forwarded_proto(r, &url_string);
- if (modified_url) {
- url.Reset(url_string);
- CHECK(url.IsWebValid()) << "The output of ps_apply_x_forwarded_proto"
- << " should always be a valid url because it only"
- << " changes the scheme between http and https.";
- }
- }
-
- bool pagespeed_resource =
- !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
- bool is_an_admin_handler =
- response_category == RequestRouting::kStatistics ||
- response_category == RequestRouting::kGlobalStatistics ||
- response_category == RequestRouting::kConsole ||
- response_category == RequestRouting::kAdmin ||
- response_category == RequestRouting::kGlobalAdmin ||
- response_category == RequestRouting::kCachePurge;
-
- // Normally if we're disabled we won't handle any requests, but if we're in
- // standby mode we do want to handle requests for .pagespeed. resources.
- if (options->unplugged() ||
- (!options->enabled() && !pagespeed_resource)) {
- // Disabled via query params or request headers.
- return NGX_DECLINED;
- }
-
- if (!html_rewrite) {
- // create request ctx
- CHECK(ctx == NULL);
- ctx = new ps_request_ctx_t();
-
- ctx->r = r;
- ctx->html_rewrite = false;
- ctx->in_place = false;
- ctx->follow_flushes = options->follow_flushes();
- ctx->preserve_caching_headers = kDontPreserveHeaders;
-
- // See build_context_for_request() in mod_instaweb.cc
- // TODO(jefftk): Is this the right place to be modifying caching headers for
- // html fetches? Or should that be done later, in the headers flow for
- // filter mode, rather than here in resource fetch mode?
- if (!options->modify_caching_headers()) {
- ctx->preserve_caching_headers = kPreserveAllCachingHeaders;
- } else if (!options->IsDownstreamCacheIntegrationEnabled()) {
- // Downstream cache integration is not enabled. Disable original
- // Cache-Control headers.
- ctx->preserve_caching_headers = kDontPreserveHeaders;
- } else if (!pagespeed_resource && !is_an_admin_handler) {
- ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
- // Downstream cache integration is enabled. If a rebeaconing key has been
- // configured and there is a ShouldBeacon header with the correct key,
- // disable original Cache-Control headers so that the instrumented page is
- // served out with no-cache.
- StringPiece should_beacon(request_headers->Lookup1(kPsaShouldBeacon));
- if (options->MatchesDownstreamCacheRebeaconingKey(should_beacon)) {
- ctx->preserve_caching_headers = kDontPreserveHeaders;
- }
- }
-
- ctx->recorder = NULL;
- ctx->url_string = url_string;
- ctx->location_field_set = false;
- ctx->psol_vary_accept_only = false;
-
- // Set up a cleanup handler on the request.
- ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
- if (cleanup == NULL) {
- ps_release_request_context(ctx);
- return NGX_ERROR;
- }
- cleanup->handler = ps_release_request_context;
- cleanup->data = ctx;
- ngx_http_set_ctx(r, ctx, ngx_pagespeed);
- }
-
- if (pagespeed_resource) {
- // TODO(jefftk): Set using_spdy appropriately. See
- // ProxyInterface::ProxyRequestCallback
- ps_create_base_fetch(url.Spec(), ctx, request_context,
- request_headers.release(), kPageSpeedResource,
- options);
- ResourceFetch::Start(
- url,
- custom_options.release() /* null if there aren't custom options */,
- cfg_s->server_context, ctx->base_fetch);
- return ps_async_wait_response(r);
- } else if (is_an_admin_handler) {
- ps_create_base_fetch(url.Spec(), ctx, request_context,
- request_headers.release(), kAdminPage, options);
- QueryParams query_params;
- query_params.ParseFromUrl(url);
-
- PosixTimer timer;
- int64 now_ms = timer.NowMs();
- ctx->base_fetch->response_headers()->SetDateAndCaching(
- now_ms, 0 /* max-age */, ", no-cache");
-
- if (response_category == RequestRouting::kStatistics ||
- response_category == RequestRouting::kGlobalStatistics) {
- cfg_s->server_context->StatisticsPage(
- response_category == RequestRouting::kGlobalStatistics,
- query_params,
- cfg_s->server_context->config(),
- ctx->base_fetch);
- } else if (response_category == RequestRouting::kConsole) {
- cfg_s->server_context->ConsoleHandler(
- *cfg_s->server_context->config(),
- AdminSite::kStatistics,
- query_params,
- ctx->base_fetch);
- } else if (response_category == RequestRouting::kAdmin ||
- response_category == RequestRouting::kGlobalAdmin) {
- cfg_s->server_context->AdminPage(
- response_category == RequestRouting::kGlobalAdmin,
- url,
- query_params,
- custom_options == NULL ? cfg_s->server_context->config()
- : custom_options.get(),
- ctx->base_fetch);
- } else if (response_category == RequestRouting::kCachePurge) {
- AdminSite* admin_site = cfg_s->server_context->admin_site();
- admin_site->PurgeHandler(url_string,
- cfg_s->server_context->cache_path(),
- ctx->base_fetch);
- } else {
- CHECK(false);
- }
-
- return ps_async_wait_response(r);
- } else if (!html_rewrite && response_category == RequestRouting::kResource) {
- bool is_proxy = false;
- GoogleString mapped_url;
- GoogleString host_header;
-
- if (options->domain_lawyer()->MapOriginUrl(
- url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
- ps_create_base_fetch(url.Spec(), ctx, request_context,
- request_headers.release(), kPageSpeedProxy, options);
-
- RewriteDriver* driver;
- if (custom_options.get() == NULL) {
- driver = cfg_s->server_context->NewRewriteDriver(
- ctx->base_fetch->request_context());
- } else {
- driver = cfg_s->server_context->NewCustomRewriteDriver(
- custom_options.release(), ctx->base_fetch->request_context());
- }
-
- driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
- driver->set_pagespeed_query_params(pagespeed_query_params);
- driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
- cfg_s->proxy_fetch_factory->StartNewProxyFetch(
- mapped_url, ctx->base_fetch, driver, NULL /*property_callback*/,
- NULL /*original_content_fetch*/);
-
- return ps_async_wait_response(r);
- }
-
- }
-
- if (html_rewrite && options->IsAllowed(url.Spec())) {
- ps_create_base_fetch(url.Spec(), ctx, request_context,
- request_headers.release(), kHtmlTransform, options);
- // Do not store driver in request_context, it's not safe.
- RewriteDriver* driver;
-
- // If we don't have custom options we can use NewRewriteDriver which reuses
- // rewrite drivers and so is faster because there's no wait to construct
- // them. Otherwise we have to build a new one every time.
-
- if (custom_options.get() == NULL) {
- driver = cfg_s->server_context->NewRewriteDriver(
- ctx->base_fetch->request_context());
- } else {
- // NewCustomRewriteDriver takes ownership of custom_options.
- driver = cfg_s->server_context->NewCustomRewriteDriver(
- custom_options.release(), ctx->base_fetch->request_context());
- }
-
- driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
- driver->set_pagespeed_query_params(pagespeed_query_params);
- driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
-
- // TODO(jefftk): FlushEarlyFlow would go here.
- ProxyFetchPropertyCallbackCollector* property_callback =
- ProxyFetchFactory::InitiatePropertyCacheLookup(
- !html_rewrite /* is_resource_fetch */,
- url,
- cfg_s->server_context,
- options,
- ctx->base_fetch);
-
- // Will call StartParse etc. The rewrite driver will take care of deleting
- // itself if necessary.
- ctx->proxy_fetch = cfg_s->proxy_fetch_factory->CreateNewProxyFetch(
- url_string, ctx->base_fetch, driver,
- property_callback,
- NULL /* original_content_fetch */);
- ctx->proxy_fetch->set_trusted_input(true);
- return NGX_OK;
- }
-
- if (options->in_place_rewriting_enabled() &&
- options->enabled() &&
- options->IsAllowed(url.Spec())) {
- ps_create_base_fetch(url.Spec(), ctx, request_context,
- request_headers.release(), kIproLookup, options);
-
- // Do not store driver in request_context, it's not safe.
- RewriteDriver* driver;
- if (custom_options.get() == NULL) {
- driver = cfg_s->server_context->NewRewriteDriver(
- ctx->base_fetch->request_context());
- } else {
- // NewCustomRewriteDriver takes ownership of custom_options.
- driver = cfg_s->server_context->NewCustomRewriteDriver(
- custom_options.release(), ctx->base_fetch->request_context());
- }
-
- driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
- ctx->driver = driver;
-
- cfg_s->server_context->message_handler()->Message(
- kInfo, "Trying to serve rewritten resource in-place: %s",
- url_string.c_str());
-
- ctx->in_place = true;
- ctx->driver->FetchInPlaceResource(
- url, false /* proxy_mode */, ctx->base_fetch);
-
- return ps_async_wait_response(r);
- }
-
- // NOTE: We are using the below debug message as is for some of our system
- // tests. So, be careful about test breakages caused by changing or
- // removing this line.
- ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0,
- "Passing on content handling for non-pagespeed resource '%s'",
- url_string.c_str());
- CHECK(ctx->base_fetch == NULL);
- // set html_rewrite flag.
- ctx->html_rewrite = true;
- return NGX_DECLINED;
-}
-
-// Send each buffer in the chain to the proxy_fetch for optimization.
-// Eventually it will make it's way, optimized, to base_fetch.
-void ps_send_to_pagespeed(ngx_http_request_t* r,
- ps_request_ctx_t* ctx,
- ps_srv_conf_t* cfg_s,
- ngx_chain_t* in) {
- ngx_chain_t* cur;
- int last_buf = 0;
- for (cur = in; cur != NULL; cur = cur->next) {
- last_buf = cur->buf->last_buf;
- // Buffers are not really the last buffer until they've been through
- // pagespeed.
- cur->buf->last_buf = 0;
-
- CHECK(ctx->proxy_fetch != NULL);
- if (ctx->inflater_ == NULL) {
- ctx->proxy_fetch->Write(
- StringPiece(reinterpret_cast(cur->buf->pos),
- cur->buf->last - cur->buf->pos), cfg_s->handler);
- } else {
- char buf[kStackBufferSize];
- ctx->inflater_->SetInput(reinterpret_cast(cur->buf->pos),
- cur->buf->last - cur->buf->pos);
- while (ctx->inflater_->HasUnconsumedInput()) {
- int num_inflated_bytes = ctx->inflater_->InflateBytes(
- buf, kStackBufferSize);
- if (num_inflated_bytes < 0) {
- cfg_s->handler->Message(kWarning, "Corrupted inflation");
- } else if (num_inflated_bytes > 0) {
- ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
- cfg_s->handler);
- }
- }
- }
- if (cur->buf->flush && ctx->follow_flushes) {
- // Calling ctx->proxy_fetch->Flush(cfg_s->handler) will be a no-op here,
- // unless we have follow_flushes or flush_html enabled. Note that PSOL
- // might aggregate multiple flushes into 1, and actually flush a little bit
- // later due to html parser state and earlier scheduled operations.
- // Also, unless we also set the flush flag on the nginx buffers we won't
- // actually flush.
- // Note that too many flushes could harm optimization over larger html
- // fragments as PSOL gets less context to work with, e.g. it can't combine
- // two css files if a flush happens in between.
- ctx->proxy_fetch->Flush(cfg_s->handler);
- }
-
- // We're done with buffers as we pass them through, so mark them as sent as
- // we go.
- cur->buf->pos = cur->buf->last;
- }
-
- if (last_buf) {
- ctx->proxy_fetch->Done(true /* success */);
- ctx->proxy_fetch = NULL; // ProxyFetch deletes itself on Done().
- }
-}
-
-#ifndef ngx_http_clear_etag
-// The ngx_http_clear_etag(r) macro was added in 1.3.3. Backport it if it's not
-// present.
-#define ngx_http_clear_etag(r) \
- if (r->headers_out.etag) { \
- r->headers_out.etag->hash = 0; \
- r->headers_out.etag = NULL; \
- }
-#endif
-
-void ps_strip_html_headers(ngx_http_request_t* r) {
- // We're modifying content, so switch to 'Transfer-Encoding: chunked' and
- // calculate on the fly.
- ngx_http_clear_content_length(r);
-
- ngx_table_elt_t* header;
- NgxListIterator it(&(r->headers_out.headers.part));
- while ((header = it.Next()) != NULL) {
- // We also need to strip:
- // Accept-Ranges
- // - won't work because our html changes
- // Vary: Accept-Encoding
- // - our gzip filter will add this later
- if (STR_CASE_EQ_LITERAL(header->key, "Accept-Ranges") ||
- (STR_CASE_EQ_LITERAL(header->key, "Vary") &&
- STR_CASE_EQ_LITERAL(header->value, "Accept-Encoding"))) {
- // Response headers with hash of 0 are excluded from the response.
- header->hash = 0;
- }
- }
-}
-
-// Returns true, if the the response headers indicate there are multiple
-// content encodings.
-bool ps_has_stacked_content_encoding(ngx_http_request_t* r) {
- ngx_uint_t i;
- ngx_list_part_t* part = &(r->headers_out.headers.part);
- ngx_table_elt_t* header = static_cast(part->elts);
- int field_count = 0;
-
- for (i = 0 ; /* void */; i++) {
- if (i >= part->nelts) {
- if (part->next == NULL) {
- break;
- }
-
- part = part->next;
- header = static_cast(part->elts);
- i = 0;
- }
-
- // Inspect Content-Encoding headers, checking all value fields
- // If an origin returns gzip,foo, that is what we will get here.
- if (STR_CASE_EQ_LITERAL(header[i].key, "Content-Encoding")) {
- if (header[i].value.data != NULL && header[i].value.len > 0) {
- char* p = reinterpret_cast(header[i].value.data);
- ngx_uint_t j;
- for (j = 0; j < header[i].value.len; j++) {
- if (p[j] == ',' || j == header[i].value.len - 1) {
- field_count++;
- }
- }
- if (field_count > 1) {
- return true;
- }
- }
- }
- }
-
- return false;
-}
-
-ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
- u_char* etag = reinterpret_cast(
- const_cast(kInternalEtagName));
- ngx_table_elt_t* header;
- NgxListIterator it(&(r->headers_out.headers.part));
- while ((header = it.Next()) != NULL) {
- if (header->key.len == strlen(kInternalEtagName) &&
- !ngx_strncasecmp(header->key.data, etag, header->key.len)) {
- header->key.data = reinterpret_cast(const_cast("ETag"));
- header->key.len = 4;
- r->headers_out.etag = header;
- break;
- }
- }
-
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-#if (NGX_HTTP_GZIP)
- if (ctx && ctx->psol_vary_accept_only) {
- r->gzip_vary = 0;
- }
-#endif
-
- if (ctx && ctx->recorder) {
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- int s_maxage_sec =
- cfg_s->server_context->global_options()->EffectiveInPlaceSMaxAgeSec();
- if (s_maxage_sec != -1) {
- GoogleString existing_cache_control;
- bool cache_control_present = ps_get_cache_control(
- r, &existing_cache_control);
- GoogleString updated_cache_control;
- if (ResponseHeaders::ApplySMaxAge(s_maxage_sec,
- existing_cache_control,
- &updated_cache_control)) {
- // We're modifing the cache control header; save a copy first.
- // NULL indicates that the header was not present.
- ctx->recorder->SaveCacheControl(
- cache_control_present ? existing_cache_control.c_str() : NULL);
-
- // Replace the cache-control with our new s-maxage-including one.
- ps_set_cache_control(r, string_piece_to_pool_string(
- r->pool, updated_cache_control));
- }
- }
- }
-
- return ngx_http_ef_next_header_filter(r);
-}
-
-namespace html_rewrite {
-ngx_http_output_header_filter_pt ngx_http_next_header_filter;
-ngx_http_output_body_filter_pt ngx_http_next_body_filter;
-
-// After pagespeed has had a chance to run, copy the headers it produced to
-// nginx so it can send them out to the browser.
-ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- if (ps_disabled(cfg_s)) {
- return ngx_http_next_header_filter(r);
- }
-
- if (r != r->main) {
- // Don't handle subrequests.
- return ngx_http_next_header_filter(r);
- }
- // Poll for cache flush on every request (polls are rate-limited).
- cfg_s->server_context->FlushCacheIfNecessary();
-
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- if (ctx == NULL || ctx->html_rewrite == false) {
- return ngx_http_next_header_filter(r);
- }
-
- if (r->err_status != 0) {
- ctx->html_rewrite = false;
- return ngx_http_next_header_filter(r);
- }
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http pagespeed html rewrite header filter \"%V\"", &r->uri);
-
- // We don't know what this request is, but we only want to send html through
- // to pagespeed. Check the content type header and find out.
- const ContentType* content_type =
- MimeTypeToContentType(
- str_to_string_piece(r->headers_out.content_type));
- if (content_type == NULL || !content_type->IsHtmlLike()) {
- // Unknown or otherwise non-html content type: skip it.
- ctx->html_rewrite = false;
- return ngx_http_next_header_filter(r);
- }
-
- ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
- RequestRouting::kResource);
- if (rc != NGX_OK) {
- ctx->html_rewrite = false;
- return ngx_http_next_header_filter(r);
- }
-
- if (r->headers_out.content_encoding &&
- r->headers_out.content_encoding->value.len) {
- // headers_out.content_encoding will be set to the exact last
- // Content-Encoding response header value that nginx receives. To
- // check if there were multiple (aka stacked) encodings in the
- // response headers, we must iterate them all.
- if (!ps_has_stacked_content_encoding(r)) {
- StringPiece content_encoding =
- str_to_string_piece(r->headers_out.content_encoding->value);
- GzipInflater::InflateType inflate_type = GzipInflater::kGzip;
- bool is_encoded = false;
- if (StringCaseEqual(content_encoding, "deflate")) {
- is_encoded = true;
- inflate_type = GzipInflater::kDeflate;
- } else if (StringCaseEqual(content_encoding, "gzip")) {
- is_encoded = true;
- }
-
- if (is_encoded) {
- r->headers_out.content_encoding->hash = 0;
- r->headers_out.content_encoding = NULL;
- ctx->inflater_ = new GzipInflater(inflate_type);
- ctx->inflater_->Init();
- }
- }
- }
-
- ps_strip_html_headers(r);
- // See https://github.com/pagespeed/ngx_pagespeed/issues/819
- ctx->location_field_set = r->headers_out.location != NULL;
-
- // TODO(jefftk): is this thread safe?
- copy_response_headers_from_ngx(r, ctx->base_fetch->response_headers());
-
- ps_set_buffered(r, true);
- r->filter_need_in_memory = 1;
- return NGX_AGAIN;
-}
-
-ngx_int_t ps_html_rewrite_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- if (ps_disabled(cfg_s)) {
- return ngx_http_next_body_filter(r, in);
- }
-
- if (r != r->main) {
- // Don't handle subrequests.
- return ngx_http_next_body_filter(r, in);
- }
- // Don't need to check for a cache flush; already did in
- // ps_html_rewrite_header_filter.
-
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- if (ctx == NULL || ctx->html_rewrite == false) {
- // ctx is null iff we've decided to pass through this request unchanged.
- return ngx_http_next_body_filter(r, in);
- }
-
- // We don't want to handle requests with errors, but we should be dealing with
- // that in the header filter and not initializing ctx.
- CHECK(r->err_status == 0); // NOLINT
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http pagespeed html rewrite body filter \"%V\"", &r->uri);
-
-
- if (in != NULL) {
- // Send all input data to the proxy fetch.
- ps_send_to_pagespeed(r, ctx, cfg_s, in);
- }
-
- return ngx_http_next_body_filter(r, NULL);
-}
-
-void ps_html_rewrite_filter_init() {
- ngx_http_next_header_filter = ngx_http_top_header_filter;
- ngx_http_top_header_filter = ps_html_rewrite_header_filter;
-
- ngx_http_next_body_filter = ngx_http_top_body_filter;
- ngx_http_top_body_filter = ps_html_rewrite_body_filter;
-}
-
-} // namespace html_rewrite
-
-using html_rewrite::ps_html_rewrite_filter_init;
-
-namespace in_place {
-
-ngx_http_output_header_filter_pt ngx_http_next_header_filter;
-ngx_http_output_body_filter_pt ngx_http_next_body_filter;
-
-// Header filter to support IPRO.
-//
-// The control flow here is tricky, probably excessively so.
-ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
-
- if (ctx == NULL) {
- return ngx_http_next_header_filter(r);
- }
-
- if (ctx->recorder != NULL) {
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ps in place check header filter recording: %V", &r->uri);
-
- CHECK(!ctx->in_place);
-
- // We didn't find this resource in cache originally, so we're recording it
- // as it passes us by. At this point the headers from things that run
- // before us are set but not things that run after us, which means here is
- // where we need to check whether there's a "Content-Encoding: gzip". If we
- // waited to do this in ps_in_place_body_filter we wouldn't be able to tell
- // the difference between response headers that have "C-E: gz" because we're
- // proxying for an upstream that gzipped the content and response headers
- // that have it because the gzip filter (which runs after us) is going to
- // produce gzipped output.
- //
- // The recorder will do this checking, so pass it the headers.
- ResponseHeaders response_headers;
- copy_response_headers_from_ngx(r, &response_headers);
- ctx->recorder->ConsiderResponseHeaders(
- InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
- return ngx_http_next_header_filter(r);
- }
-
- if (!ctx->in_place) {
- return ngx_http_next_header_filter(r);
- }
-
- // If our request was finalized during the IPRO lookup, we decline.
- if (r->connection->error) {
- return ps_decline_request(r);
- }
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ps in place check header filter initial: %V", &r->uri);
-
- int status_code = r->headers_out.status;
- bool status_ok = (status_code != 0) && (status_code < 400);
-
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- NgxServerContext* server_context = cfg_s->server_context;
- MessageHandler* message_handler = cfg_s->handler;
- GoogleString url = ps_determine_url(r);
- // The URL we use for cache key is a bit different since it may
- // have PageSpeed query params removed.
- GoogleString cache_url = ctx->url_string;
-
- // continue process
- if (status_ok) {
- ctx->in_place = false;
-
- server_context->rewrite_stats()->ipro_served()->Add(1);
- message_handler->Message(
- kInfo, "Serving rewritten resource in-place: %s",
- url.c_str());
-
- return ngx_http_next_header_filter(r);
- }
-
- if (status_code == CacheUrlAsyncFetcher::kNotInCacheStatus &&
- !r->header_only) {
- server_context->rewrite_stats()->ipro_not_in_cache()->Add(1);
- server_context->message_handler()->Message(
- kInfo,
- "Could not rewrite resource in-place "
- "because URL is not in cache: %s",
- cache_url.c_str());
- const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
- ctx->driver->options());
-
- RequestContextPtr request_context(
- cfg_s->server_context->NewRequestContext(r));
- request_context->set_options(options->ComputeHttpOptions());
- RequestHeaders request_headers;
- copy_request_headers_from_ngx(r, &request_headers);
- // This URL was not found in cache (neither the input resource nor
- // a ResourceNotCacheable entry) so we need to get it into cache
- // (or at least a note that it cannot be cached stored there).
- // We do that using an Apache output filter.
- ctx->recorder = new InPlaceResourceRecorder(
- request_context,
- cache_url,
- ctx->driver->CacheFragment(),
- request_headers.GetProperties(),
- options->ipro_max_response_bytes(),
- options->ipro_max_concurrent_recordings(),
- server_context->http_cache(),
- server_context->statistics(),
- message_handler);
- // set in memory flag for in_place_body_filter
- r->filter_need_in_memory = 1;
-
- // We don't have the response headers at all yet because we haven't yet gone
- // to the backend.
- } else {
- server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
- message_handler->Message(
- kInfo, "Could not rewrite resource in-place: %s", url.c_str());
- }
-
- return ps_decline_request(r);
-}
-
-// If we've decided that we should record this response for future optimization
-// with IPRO, then log the bytes as they come through
-ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- if (ctx == NULL || ctx->recorder == NULL) {
- return ngx_http_next_body_filter(r, in);
- }
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ps in place body filter: %V", &r->uri);
-
- InPlaceResourceRecorder* recorder = ctx->recorder;
- for (ngx_chain_t* cl = in; cl; cl = cl->next) {
- if (ngx_buf_size(cl->buf)) {
- CHECK(ngx_buf_in_memory(cl->buf));
- StringPiece contents(reinterpret_cast(cl->buf->pos),
- ngx_buf_size(cl->buf));
- recorder->Write(contents, recorder->handler());
- }
-
- if (cl->buf->flush) {
- recorder->Flush(recorder->handler());
- }
-
- if (cl->buf->last_buf || recorder->failed()) {
- ResponseHeaders response_headers;
- copy_response_headers_from_ngx(r, &response_headers);
- ctx->recorder->DoneAndSetHeaders(
- &response_headers,
- cl->buf->last_buf /* response is complete if last_buf is set */);
- ctx->recorder = NULL;
- break;
- }
- }
-
- return ngx_http_next_body_filter(r, in);
-}
-
-void ps_in_place_filter_init() {
- ngx_http_next_header_filter = ngx_http_top_header_filter;
- ngx_http_top_header_filter = ps_in_place_check_header_filter;
-
- ngx_http_next_body_filter = ngx_http_top_body_filter;
- ngx_http_top_body_filter = ps_in_place_body_filter;
-}
-
-} // namespace in_place
-
-using in_place::ps_in_place_filter_init;
-
-ngx_int_t send_out_headers_and_body(
- ngx_http_request_t* r,
- const ResponseHeaders& response_headers,
- const GoogleString& output) {
- ngx_int_t rc = copy_response_headers_to_ngx(
- r, response_headers, kDontPreserveHeaders);
-
- if (rc != NGX_OK) {
- return NGX_ERROR;
- }
-
- rc = ngx_http_send_header(r);
-
- if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
- return rc;
- }
-
- // Send the body.
- ngx_chain_t* out;
- rc = string_piece_to_buffer_chain(
- r->pool, output, &out, true /* send_last_buf */, false);
- if (rc == NGX_ERROR) {
- return NGX_ERROR;
- }
- CHECK(rc == NGX_OK);
-
- return ngx_http_output_filter(r, out);
-}
-
-// Handle responses where we have the content we need in memory and can just
-// send it right out.
-ngx_int_t ps_simple_handler(ngx_http_request_t* r,
- NgxServerContext* server_context,
- RequestRouting::Response response_category) {
- NgxRewriteDriverFactory* factory =
- static_cast(
- server_context->factory());
- NgxMessageHandler* message_handler = factory->ngx_message_handler();
- StringPiece request_uri_path = str_to_string_piece(r->uri);
-
- GoogleString url_string = ps_determine_url(r);
- GoogleUrl url(url_string);
- QueryParams query_params;
- if (url.IsWebValid()) {
- query_params.ParseFromUrl(url);
- }
-
- GoogleString output;
- StringWriter writer(&output);
- HttpStatus::Code status = HttpStatus::kOK;
- ContentType content_type = kContentTypeHtml;
- StringPiece cache_control = HttpAttributes::kNoCache;
- const char* error_message = NULL;
-
- switch (response_category) {
- case RequestRouting::kStaticContent: {
- StringPiece file_contents;
- if (!server_context->static_asset_manager()->GetAsset(
- request_uri_path.substr(factory->static_asset_prefix().length()),
- &file_contents, &content_type, &cache_control)) {
- return NGX_DECLINED;
- }
- file_contents.CopyToString(&output);
- break;
- }
- case RequestRouting::kMessages: {
- GoogleString log;
- StringWriter log_writer(&log);
- if (!message_handler->Dump(&log_writer)) {
- writer.Write("Writing to ngx_pagespeed_message failed. \n"
- "Please check if it's enabled in pagespeed.conf.\n",
- message_handler);
- } else {
- HtmlKeywords::WritePre(log, "", &writer, message_handler);
- }
- break;
- }
- default:
- ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
- "ps_simple_handler: unknown RequestRouting.");
- return NGX_ERROR;
- }
-
- if (error_message != NULL) {
- status = HttpStatus::kNotFound;
- content_type = kContentTypeHtml;
- output = error_message;
- }
-
- ResponseHeaders response_headers;
- response_headers.SetStatusAndReason(status);
- response_headers.set_major_version(1);
- response_headers.set_minor_version(1);
-
- response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
- // http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
- // Script and styleSheet elements will reject responses with
- // incorrect MIME types if the server sends the response header
- // "X-Content-Type-Options: nosniff". This is a security feature
- // that helps prevent attacks based on MIME-type confusion.
- response_headers.Add("X-Content-Type-Options", "nosniff");
-
- int64 now_ms = factory->timer()->NowMs();
- response_headers.SetDate(now_ms);
- response_headers.SetLastModified(now_ms);
- response_headers.Add(HttpAttributes::kCacheControl, cache_control);
-
- char* cache_control_s = string_piece_to_pool_string(r->pool, cache_control);
- if (cache_control_s != NULL) {
- if (FindIgnoreCase(cache_control, "private") == StringPiece::npos) {
- response_headers.Add(HttpAttributes::kEtag, "W/\"0\"");
- }
- }
-
- return send_out_headers_and_body(r, response_headers, output);
-}
-
-void ps_beacon_handler_helper(ngx_http_request_t* r,
- StringPiece beacon_data) {
- ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ps_beacon_handler_helper: beacon[%d] %*s",
- beacon_data.size(), beacon_data.size(),
- beacon_data.data());
-
- StringPiece user_agent;
- if (r->headers_in.user_agent != NULL) {
- user_agent = str_to_string_piece(r->headers_in.user_agent->value);
- }
-
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- CHECK(cfg_s != NULL);
-
- RequestContextPtr request_context(
- cfg_s->server_context->NewRequestContext(r));
- // TODO(sligocki): Do we want custom options here? It probably doesn't matter
- // for beacons.
- request_context->set_options(
- cfg_s->server_context->global_options()->ComputeHttpOptions());
-
- cfg_s->server_context->HandleBeacon(beacon_data,
- user_agent,
- request_context);
-
- ps_set_cache_control(r, const_cast("max-age=0, no-cache"));
-
- // TODO(jefftk): figure out how to insert Content-Length:0 as a response
- // header so wget doesn't hang.
-}
-
-// Load the request body into out. ngx_http_read_client_request_body must
-// already have been called. Return false on failure, true on success.
-bool ps_request_body_to_string_piece(
- ngx_http_request_t* r, StringPiece* out) {
- if (r->request_body == NULL || r->request_body->bufs == NULL) {
- ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
- "ps_request_body_to_string_piece: "
- "empty request body.");
- return false;
- }
-
- if (r->request_body->temp_file) {
- u_char buf[POST_BUF_READ_SIZE];
- int count = 0;
- ssize_t ret;
- GoogleString tmp;
-
- // Note that we depend on nginx to impose sensible limits on post data.
- while ((ret = ngx_read_file(&r->request_body->temp_file->file,
- buf, POST_BUF_READ_SIZE, count)) > 0) {
- tmp.append(reinterpret_cast(buf), ret);
- count += ret;
- }
-
- if (ret < 0) {
- ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
- "ps_request_body_to_string_piece: "
- "error reading post body.");
- return false;
- }
-
- // We have read the complete post body, copy it to a buffer
- // from the request's pool.
- u_char* data = reinterpret_cast(ngx_palloc(r->pool, count));
- memcpy(data, tmp.c_str(), count);
- *out = StringPiece(reinterpret_cast(data), count);
-
- return true;
- } else if (r->request_body->bufs->next == NULL) {
- // There's just one buffer, so we can simply return a StringPiece pointing
- // to this buffer.
- ngx_buf_t* buffer = r->request_body->bufs->buf;
- CHECK(!buffer->in_file);
- int len = buffer->last - buffer->pos;
- ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ngx_pagespeed beacon: single buffer of %d", len);
- *out = StringPiece(reinterpret_cast(buffer->pos), len);
- return true;
- } else {
- // There are multiple buffers, so we need to allocate memory for a string to
- // hold the whole result. This should only happen when the POST is sent
- // with "Transfer-Encoding: Chunked".
-
- // First determine how much data there is.
- int len = 0;
- int buffers = 0;
-
- ngx_chain_t* chain_link;
- for (chain_link = r->request_body->bufs;
- chain_link != NULL;
- chain_link = chain_link->next) {
- len += chain_link->buf->last - chain_link->buf->pos;
- buffers++;
- }
-
- ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "ngx_pagespeed beacon: %d buffers totalling %d", len);
-
- // Allocate a string to store the combined result.
- u_char* s = static_cast(ngx_palloc(r->pool, len));
- if (s == NULL) {
- ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
- "ps_request_body_to_string_piece: "
- "failed to allocate memory");
- return false;
- }
-
- // Copy the data into the combined string.
- u_char* current_position = s;
- int i;
- for (chain_link = r->request_body->bufs, i = 0;
- chain_link != NULL;
- chain_link = chain_link->next, i++) {
- ngx_buf_t* buffer = chain_link->buf;
- CHECK(!buffer->in_file);
- current_position = ngx_copy(current_position, buffer->pos,
- buffer->last - buffer->pos);
- }
- CHECK_EQ(current_position, s + len);
- *out = StringPiece(reinterpret_cast(s), len);
- return true;
- }
-}
-
-// Parses out query params from the request.
-void ps_query_params_handler(ngx_http_request_t* r, StringPiece* data) {
- StringPiece unparsed_uri = str_to_string_piece(r->unparsed_uri);
- stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
- if (question_mark_index == StringPiece::npos) {
- *data = "";
- } else {
- *data = unparsed_uri.substr(
- question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
- }
-}
-
-// Called after nginx reads the request body from the client. For another
-// example processing request buffers, see ngx_http_form_input_module.c
-void ps_beacon_body_handler(ngx_http_request_t* r) {
- // Even if the beacon is a POST, the originating url should be in the query
- // params, not the POST body.
- StringPiece query_param_beacon_data;
- ps_query_params_handler(r, &query_param_beacon_data);
-
- StringPiece request_body;
- bool ok = ps_request_body_to_string_piece(r, &request_body);
- GoogleString beacon_data = StrCat(
- query_param_beacon_data, "&", request_body);
- if (ok) {
- ps_beacon_handler_helper(r, beacon_data.c_str());
- ngx_http_finalize_request(r, NGX_HTTP_NO_CONTENT);
- } else {
- ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
- }
-}
-
-// We need to get the beacon data to ps_beacon_body_handler so it can pass it
-// along to SystemServerContext::HandleBeacon, but it might have been POSTed.
-// If it's posted we need to make an async call to get the data, otherwise just
-// read it out of the query params.
-ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
- if (r->method == NGX_HTTP_POST) {
- // Use post body. Handler functions are called before the request body has
- // been read from the client, so we need to ask nginx to read it from the
- // client and then call us back. Control flow continues in
- // ps_beacon_body_handler unless there's an error reading the request body.
- //
- // See: http://forum.nginx.org/read.php?2,31312,31312
- ngx_int_t rc = ngx_http_read_client_request_body(r, ps_beacon_body_handler);
- if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {
- return rc;
- }
- return NGX_DONE;
- } else {
- // Use query params.
- StringPiece query_param_beacon_data;
- ps_query_params_handler(r, &query_param_beacon_data);
- ps_beacon_handler_helper(r, query_param_beacon_data);
- return NGX_HTTP_NO_CONTENT;
- }
-}
-
-// Some things pagespeed filters on the way past (html) and other things it
-// actually handles, like requests for resources
-// (example.css.pagespeed.ce.LyfcM6Wulf.css) and static content
-// (/ngx_pagespeed_static/js_defer.q1EBmcgYOC.js). This is called once we know
-// we need to handle a resource, and it figures out which particular handler can
-// supply it to the user.
-ngx_int_t ps_content_handler(ngx_http_request_t* r) {
- ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
- if (ps_disabled(cfg_s)) {
- // Pagespeed is not on for this server block.
- return NGX_DECLINED;
- }
-
- // Poll for cache flush on every request (polls are rate-limited).
- cfg_s->server_context->FlushCacheIfNecessary();
-
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "http pagespeed handler \"%V\"", &r->uri);
-
- RequestRouting::Response response_category =
- ps_route_request(r);
- switch (response_category) {
- case RequestRouting::kError:
- return NGX_ERROR;
- case RequestRouting::kPagespeedDisabled:
- case RequestRouting::kInvalidUrl:
- case RequestRouting::kPagespeedSubrequest:
- case RequestRouting::kErrorResponse:
- return NGX_DECLINED;
- case RequestRouting::kBeacon:
- return ps_beacon_handler(r);
- case RequestRouting::kStaticContent:
- case RequestRouting::kMessages:
- return ps_simple_handler(r, cfg_s->server_context, response_category);
- case RequestRouting::kStatistics:
- case RequestRouting::kGlobalStatistics:
- case RequestRouting::kConsole:
- case RequestRouting::kAdmin:
- case RequestRouting::kGlobalAdmin:
- case RequestRouting::kCachePurge:
- case RequestRouting::kResource:
- return ps_resource_handler(
- r, false /* html rewrite */, response_category);
- }
-
- CHECK(0);
- return NGX_ERROR;
-}
-
-ngx_int_t ps_phase_handler(ngx_http_request_t* r,
- ngx_http_phase_handler_t* ph) {
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "pagespeed phase: %ui", r->phase_handler);
-
- r->write_event_handler = ngx_http_request_empty_handler;
-
- ngx_int_t rc = ps_content_handler(r);
- // Warning: this requires ps_content_handler to always return NGX_DECLINED
- // directly if it's not going to handle the request. It is not ok for
- // ps_content_handler to asynchronously determine whether to handle the
- // request, returning NGX_DONE here.
- if (rc == NGX_DECLINED) {
- r->write_event_handler = ngx_http_core_run_phases;
- r->phase_handler++;
- return NGX_AGAIN;
- }
-
- ngx_http_finalize_request(r, rc);
- return NGX_OK;
-}
-
-namespace fix_headers {
-ngx_http_output_header_filter_pt ngx_http_next_header_filter;
-
-// Clear a few headers from html responses.
-
-ngx_int_t ps_html_rewrite_fix_headers_filter(ngx_http_request_t* r) {
- ps_request_ctx_t* ctx = ps_get_request_context(r);
- if (r != r->main || ctx == NULL || !ctx->html_rewrite ||
- ctx->preserve_caching_headers == kPreserveAllCachingHeaders) {
- return ngx_http_next_header_filter(r);
- }
- if (ctx->preserve_caching_headers == kDontPreserveHeaders) {
- // Don't cache html. See mod_instaweb:instaweb_fix_headers_filter.
- NgxCachingHeaders caching_headers(r);
- ps_set_cache_control(r, string_piece_to_pool_string(
- r->pool, caching_headers.GenerateDisabledCacheControl()));
- }
-
- // Pagespeed html doesn't need etags: it should never be cached.
- ngx_http_clear_etag(r);
-
- // An html page may change without the underlying file changing, because of
- // how resources are included. Pagespeed adds cache control headers for
- // resources instead of using the last modified header.
- ngx_http_clear_last_modified(r);
-
- // Clear expires
- if (r->headers_out.expires) {
- r->headers_out.expires->hash = 0;
- r->headers_out.expires = NULL;
- }
-
- return ngx_http_next_header_filter(r);
-}
-
-void ps_html_rewrite_fix_headers_filter_init() {
- ngx_http_next_header_filter = ngx_http_top_header_filter;
- ngx_http_top_header_filter = ps_html_rewrite_fix_headers_filter;
-}
-
-} // namespace fix_headers
-
-using fix_headers::ps_html_rewrite_fix_headers_filter_init;
-
-
-// preaccess_handler should be at generic phase before try_files
-ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
- ngx_http_core_main_conf_t* cmcf;
- ngx_http_phase_handler_t* ph;
- ngx_uint_t i;
-
- cmcf = static_cast(
- ngx_http_get_module_main_conf(r, ngx_http_core_module));
-
- ph = cmcf->phase_engine.handlers;
-
- i = r->phase_handler;
-
- // move handlers before try_files && content phase
- // As of nginx 1.13.4 we will be right before the try_files module
- #if (nginx_version < NGINX_1_13_4)
- while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
- ph[i + 1].checker != ngx_http_core_content_phase) {
- ph[i] = ph[i + 1];
- ph[i].next--;
- i++;
- }
- #endif
-
- // insert ps phase handler
- ph[i].checker = ps_phase_handler;
- ph[i].handler = NULL;
- ph[i].next = i + 1;
-
- // next preaccess handler
- r->phase_handler--;
- return NGX_DECLINED;
-}
-
-ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
- if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
- ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
- ngx_http_top_header_filter = ps_etag_header_filter;
- }
- return NGX_OK;
-}
-
-// Called before configuration.
-ngx_int_t ps_pre_init(ngx_conf_t* cf) {
- // Setup an intervention setter for gzip configuration and check
- // gzip configuration command signatures.
- g_gzip_setter.Init(cf);
- return NGX_OK;
-}
-
-ngx_int_t ps_init(ngx_conf_t* cf) {
- // Only put register pagespeed code to run if there was a "pagespeed"
- // configuration option set in the config file. With "pagespeed off" we
- // consider every request and choose not to do anything, while with no
- // "pagespeed" directives we won't have any effect after nginx is done loading
- // its configuration.
-
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-
- // The driver factory is on the main config and is non-NULL iff there is a
- // pagespeed configuration option in the main config or a server block. Note
- // that if any server block has pagespeed 'on' then our header filter, body
- // filter, and content handler will run in every server block. This is ok,
- // because they will notice that the server context is NULL and do nothing.
- if (cfg_m->driver_factory != NULL) {
- // The filter init order is important.
- ps_in_place_filter_init();
-
- ps_html_rewrite_fix_headers_filter_init();
- ps_base_fetch::ps_base_fetch_filter_init();
- ps_html_rewrite_filter_init();
-
- ngx_http_core_main_conf_t* cmcf = static_cast(
- ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module));
-
- int phase = NGX_HTTP_PREACCESS_PHASE;
-
- // As of nginx 1.13.4, try_files has changed.
- // https://github.com/nginx/nginx/commit/129b06dc5dfab7b4513a4f274b3778cd9b8a6a22
-#if (nginx_version >= NGINX_1_13_4)
- phase = NGX_HTTP_PRECONTENT_PHASE;
-#endif
-
- ngx_http_handler_pt* h = static_cast(
- ngx_array_push(&cmcf->phases[phase].handlers));
-
- if (h == NULL) {
- return NGX_ERROR;
- }
- *h = ps_preaccess_handler;
- }
-
- return NGX_OK;
-}
-
-ngx_http_module_t ps_etag_filter_module = {
- NULL, // preconfiguration
- ps_etag_filter_init, // postconfiguration
- NULL,
- NULL, // initialize main configuration
- NULL,
- NULL,
- NULL,
- NULL
-};
-
-ngx_http_module_t ps_module = {
- ps_pre_init, // preconfiguration
- ps_init, // postconfiguration
-
- ps_create_main_conf,
- NULL, // initialize main configuration
-
- ps_create_srv_conf,
- ps_merge_srv_conf,
-
- ps_create_loc_conf,
- ps_merge_loc_conf
-};
-
-// called after configuration is complete, but before nginx starts forking
-ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
-
- // See https://github.com/pagespeed/ngx_pagespeed/issues/1220
- if (cfg_m == NULL) {
- return NGX_OK;
- }
-
- ngx_http_core_main_conf_t* cmcf = static_cast(
- ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
- ngx_http_core_srv_conf_t** cscfp = static_cast(
- cmcf->servers.elts);
- ngx_uint_t s;
-
- std::vector server_contexts;
- // Iterate over all configured server{} blocks to collect the server contexts.
- for (s = 0; s < cmcf->servers.nelts; s++) {
- ps_srv_conf_t* cfg_s = static_cast(
- cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
- if (cfg_s->server_context != NULL) {
- server_contexts.push_back(cfg_s->server_context);
- }
- }
-
- GoogleString error_message;
- int error_index = -1;
- Statistics* global_statistics = NULL;
- cfg_m->driver_factory->PostConfig(
- server_contexts, &error_message, &error_index, &global_statistics);
- if (error_index != -1) {
- server_contexts[error_index]->message_handler()->Message(
- kError, "ngx_pagespeed is enabled. %s", error_message.c_str());
- return NGX_ERROR;
- }
-
- if (!server_contexts.empty()) {
- // TODO(oschaaf): this ignores sigpipe messages from memcached.
- // however, it would be better to not have those signals generated
- // in the first place, as suppressing them this way may interfere
- // with other modules that actually are interested in these signals
- ps_ignore_sigpipe();
-
- // If no shared-mem statistics are enabled, then init using the default
- // NullStatistics.
- if (global_statistics == NULL) {
- NgxRewriteDriverFactory::InitStats(cfg_m->driver_factory->statistics());
- }
-
- ngx_http_core_loc_conf_t* clcf = static_cast(
- ngx_http_conf_get_module_loc_conf((*cscfp), ngx_http_core_module));
-
- cfg_m->driver_factory->set_resolver(clcf->resolver);
- cfg_m->driver_factory->set_resolver_timeout(clcf->resolver_timeout);
-
- if (!cfg_m->driver_factory->CheckResolver()) {
- cfg_m->handler->Message(
- kError,
- "UseNativeFetcher is on, please configure a resolver.");
- return NGX_ERROR;
- }
-
- cfg_m->driver_factory->LoggingInit(cycle->log, true);
- cfg_m->driver_factory->RootInit();
- } else {
- delete cfg_m->driver_factory;
- cfg_m->driver_factory = NULL;
- active_driver_factory = NULL;
- }
- return NGX_OK;
-}
-
-void ps_exit_child_process(ngx_cycle_t* cycle) {
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
- NgxBaseFetch::Terminate();
- if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
- cfg_m->driver_factory->ShutDown();
- }
-}
-
-// Called when nginx forks worker processes. No threads should be started
-// before this.
-ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
- ps_main_conf_t* cfg_m = static_cast(
- ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
- if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
- return NGX_OK;
- }
-
- if (!NgxBaseFetch::Initialize(cycle)) {
- return NGX_ERROR;
- }
-
- // ChildInit() will initialise all ServerContexts, which we need to
- // create ProxyFetchFactories below
- cfg_m->driver_factory->LoggingInit(cycle->log, true);
- cfg_m->driver_factory->ChildInit();
-
- ngx_http_core_main_conf_t* cmcf = static_cast(
- ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
- ngx_http_core_srv_conf_t** cscfp = static_cast(
- cmcf->servers.elts);
- ngx_uint_t s;
-
- // Iterate over all configured server{} blocks, and find our context in it,
- // so we can create and set a ProxyFetchFactory for it.
- for (s = 0; s < cmcf->servers.nelts; s++) {
- ps_srv_conf_t* cfg_s = static_cast(
- cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
- // Some server{} blocks may not have a ServerContext in that case we must
- // not instantiate a ProxyFetchFactory.
- if (cfg_s->server_context != NULL) {
- cfg_s->proxy_fetch_factory = new ProxyFetchFactory(cfg_s->server_context);
- ngx_http_core_loc_conf_t* clcf = static_cast(
- cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]);
- cfg_m->driver_factory->SetServerContextMessageHandler(
- cfg_s->server_context, clcf->error_log);
- }
- }
-
- cfg_m->driver_factory->StartThreads();
- return NGX_OK;
-}
-
-} // namespace
-
-} // namespace net_instaweb
-
-ngx_module_t ngx_pagespeed_etag_filter = {
- NGX_MODULE_V1,
- &net_instaweb::ps_etag_filter_module,
- NULL,
- NGX_HTTP_MODULE,
- NULL,
- NULL,
- NULL,
- NULL,
- NULL,
- NULL,
- NULL,
- NGX_MODULE_V1_PADDING
-};
-
-ngx_module_t ngx_pagespeed = {
- NGX_MODULE_V1,
- &net_instaweb::ps_module,
- net_instaweb::ps_commands,
- NGX_HTTP_MODULE,
- NULL,
- net_instaweb::ps_init_module,
- net_instaweb::ps_init_child_process,
- NULL,
- NULL,
- net_instaweb::ps_exit_child_process,
- NULL,
- NGX_MODULE_V1_PADDING
-};
diff --git a/lib/ngx_pagespeed/src/ngx_pagespeed.h b/lib/ngx_pagespeed/src/ngx_pagespeed.h
deleted file mode 100644
index ae46004..0000000
--- a/lib/ngx_pagespeed/src/ngx_pagespeed.h
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#ifndef NGX_PAGESPEED_H_
-#define NGX_PAGESPEED_H_
-
-// We might be compiled with syslog.h, which #defines LOG_INFO and LOG_WARNING
-// as ints. But logging.h assumes they're usable as names, within their
-// namespace, so we need to #undef them before including logging.h
-#ifdef LOG_INFO
-#undef LOG_INFO
-#endif
-#ifdef LOG_WARNING
-#undef LOG_WARNING
-#endif
-
-extern "C" {
- #include
-}
-
-#include "base/logging.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/http/response_headers.h"
-
-namespace net_instaweb {
-
-class GzipInflater;
-class NgxBaseFetch;
-class ProxyFetch;
-class RewriteDriver;
-class RequestHeaders;
-class ResponseHeaders;
-class InPlaceResourceRecorder;
-
-// Allocate chain links and buffers from the supplied pool, and copy over the
-// data from the string piece. If the string piece is empty, return
-// NGX_DECLINED immediately unless send_last_buf.
-ngx_int_t string_piece_to_buffer_chain(
- ngx_pool_t* pool, StringPiece sp,
- ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
-
-StringPiece str_to_string_piece(ngx_str_t s);
-
-// s1: ngx_str_t, s2: string literal
-// true if they're equal, false otherwise
-#define STR_EQ_LITERAL(s1, s2) \
- ((s1).len == (sizeof(s2)-1) && \
- ngx_strncmp((s1).data, (s2), (sizeof(s2)-1)) == 0)
-
-// s1: ngx_str_t, s2: string literal
-// true if they're equal ignoring case, false otherwise
-#define STR_CASE_EQ_LITERAL(s1, s2) \
- ((s1).len == (sizeof(s2)-1) && \
- ngx_strncasecmp((s1).data, ( \
- reinterpret_cast( \
- const_cast(s2))), \
- (sizeof(s2)-1)) == 0)
-
-// Allocate memory out of the pool for the string piece, and copy the contents
-// over. Returns NULL if we can't get memory.
-char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
-
-enum PreserveCachingHeaders {
- kPreserveAllCachingHeaders, // Cache-Control, ETag, Last-Modified, etc
- kPreserveOnlyCacheControl, // Only Cache-Control.
- kDontPreserveHeaders,
-};
-
-typedef struct {
- NgxBaseFetch* base_fetch;
-
- ngx_http_request_t* r;
-
- bool html_rewrite;
- bool in_place;
-
- PreserveCachingHeaders preserve_caching_headers;
-
- // for html rewrite
- ProxyFetch* proxy_fetch;
- GzipInflater* inflater_;
-
- // for in place resource
- RewriteDriver* driver;
- InPlaceResourceRecorder* recorder;
- ResponseHeaders* ipro_response_headers;
-
- // We need to remember the URL here as well since we may modify what NGX
- // gets by stripping our special query params and honoring X-Forwarded-Proto.
- GoogleString url_string;
-
- // We need to remember if the upstream had headers_out->location set, because
- // we should mirror that when we write it back. nginx may absolutify
- // Location: headers that start with '/' without regarding X-Forwarded-Proto.
- bool location_field_set;
- bool psol_vary_accept_only;
- bool follow_flushes;
-} ps_request_ctx_t;
-
-ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
-
-void copy_request_headers_from_ngx(const ngx_http_request_t* r,
- RequestHeaders* headers);
-
-void copy_response_headers_from_ngx(const ngx_http_request_t* r,
- ResponseHeaders* headers);
-
-ngx_int_t copy_response_headers_to_ngx(
- ngx_http_request_t* r,
- const ResponseHeaders& pagespeed_headers,
- PreserveCachingHeaders preserve_caching_headers);
-
-StringPiece ps_determine_host(ngx_http_request_t* r);
-
-namespace ps_base_fetch {
-
-ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
-
-} // namespace ps_base_fetch
-
-} // namespace net_instaweb
-
-#endif // NGX_PAGESPEED_H_
diff --git a/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.cc b/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.cc
deleted file mode 100644
index bed9093..0000000
--- a/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.cc
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#include "ngx_rewrite_driver_factory.h"
-
-#include
-
-#include "log_message_handler.h"
-#include "ngx_message_handler.h"
-#include "ngx_rewrite_options.h"
-#include "ngx_server_context.h"
-#include "ngx_url_async_fetcher.h"
-
-#include "net/instaweb/http/public/rate_controller.h"
-#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
-#include "net/instaweb/http/public/wget_url_fetcher.h"
-#include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
-#include "net/instaweb/rewriter/public/server_context.h"
-#include "net/instaweb/util/public/property_cache.h"
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/null_shared_mem.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/base/stdio_file_system.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/thread_system.h"
-#include "pagespeed/kernel/http/content_type.h"
-#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
-#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
-#include "pagespeed/kernel/thread/pthread_shared_mem.h"
-#include "pagespeed/kernel/thread/scheduler_thread.h"
-#include "pagespeed/kernel/thread/slow_worker.h"
-#include "pagespeed/system/in_place_resource_recorder.h"
-#include "pagespeed/system/serf_url_async_fetcher.h"
-#include "pagespeed/system/system_caches.h"
-#include "pagespeed/system/system_rewrite_options.h"
-
-namespace net_instaweb {
-
-class FileSystem;
-class Hasher;
-class MessageHandler;
-class Statistics;
-class Timer;
-class UrlAsyncFetcher;
-class UrlFetcher;
-class Writer;
-
-class SharedCircularBuffer;
-
-NgxRewriteDriverFactory::NgxRewriteDriverFactory(
- const ProcessContext& process_context,
- SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
- : SystemRewriteDriverFactory(process_context, system_thread_system,
- NULL /* default shared memory runtime */, hostname, port),
- threads_started_(false),
- ngx_message_handler_(
- new NgxMessageHandler(timer(), thread_system()->NewMutex())),
- ngx_html_parse_message_handler_(
- new NgxMessageHandler(timer(), thread_system()->NewMutex())),
- log_(NULL),
- resolver_timeout_(NGX_CONF_UNSET_MSEC),
- use_native_fetcher_(false),
- // 100 Aligns to nginx's server-side default.
- native_fetcher_max_keepalive_requests_(100),
- ngx_shared_circular_buffer_(NULL),
- hostname_(hostname.as_string()),
- port_(port),
- process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
- process_script_variables_set_(false),
- shut_down_(false) {
- InitializeDefaultOptions();
- default_options()->set_beacon_url("/ngx_pagespeed_beacon");
- SystemRewriteOptions* system_options = dynamic_cast(
- default_options());
- system_options->set_file_cache_clean_inode_limit(500000);
- system_options->set_avoid_renaming_introspective_javascript(true);
- set_message_handler(ngx_message_handler_);
- set_html_parse_message_handler(ngx_html_parse_message_handler_);
-}
-
-NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
- ShutDown();
- ngx_shared_circular_buffer_ = NULL;
- STLDeleteElements(&uninitialized_server_contexts_);
-}
-
-Hasher* NgxRewriteDriverFactory::NewHasher() {
- return new MD5Hasher;
-}
-
-UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
- SystemRewriteOptions* config) {
- if (use_native_fetcher_) {
- NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
- config->fetcher_proxy().c_str(),
- log_,
- resolver_timeout_,
- config->blocking_fetch_timeout_ms(),
- resolver_,
- native_fetcher_max_keepalive_requests_,
- thread_system(),
- message_handler());
- ngx_url_async_fetchers_.push_back(fetcher);
- return fetcher;
- } else {
- return SystemRewriteDriverFactory::AllocateFetcher(config);
- }
-}
-
-MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
- return ngx_html_parse_message_handler_;
-}
-
-MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
- return ngx_message_handler_;
-}
-
-FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
- return new StdioFileSystem();
-}
-
-Timer* NgxRewriteDriverFactory::DefaultTimer() {
- return new PosixTimer;
-}
-
-NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
- CHECK(false);
- return NULL;
-}
-
-RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
- NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
- // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
- // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
- // -- if you use that instead then ngx_pagespeed doesn't actually end up
- // defaulting CoreFilters.
- // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
- options->SetRewriteLevel(RewriteOptions::kCoreFilters);
- return options;
-}
-
-RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
- return new NgxRewriteOptions(thread_system());
-}
-
-bool NgxRewriteDriverFactory::CheckResolver() {
- if (use_native_fetcher_ && resolver_ == NULL) {
- return false;
- }
- return true;
-}
-
-NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
- StringPiece hostname, int port) {
- NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
- uninitialized_server_contexts_.insert(server_context);
- return server_context;
-}
-
-ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
- ServerContext* sc = new NgxServerContext(this, hostname_, port_);
- InitStubDecodingServerContext(sc);
- return sc;
-}
-
-ServerContext* NgxRewriteDriverFactory::NewServerContext() {
- LOG(DFATAL) << "MakeNgxServerContext should be used instead";
- return NULL;
-}
-
-void NgxRewriteDriverFactory::ShutDown() {
- if (!shut_down_) {
- shut_down_ = true;
- SystemRewriteDriverFactory::ShutDown();
- }
-}
-
-void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
- ngx_message_handler_->set_buffer(NULL);
- ngx_html_parse_message_handler_->set_buffer(NULL);
- for (NgxMessageHandlerSet::iterator p =
- server_context_message_handlers_.begin();
- p != server_context_message_handlers_.end(); ++p) {
- (*p)->set_buffer(NULL);
- }
- server_context_message_handlers_.clear();
-}
-
-void NgxRewriteDriverFactory::StartThreads() {
- if (threads_started_) {
- return;
- }
- // TODO(jefftk): use a native nginx timer instead of running our own thread.
- // See issue #111.
- SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
- bool ok = thread->Start();
- CHECK(ok) << "Unable to start scheduler thread";
- defer_cleanup(thread->MakeDeleter());
- threads_started_ = true;
-}
-
-void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
- // Propagate process-scope options from the copy we had during nginx option
- // parsing to our own.
- if (main_options != NULL) {
- default_options()->MergeOnlyProcessScopeOptions(*main_options);
- }
-}
-
-void NgxRewriteDriverFactory::LoggingInit(
- ngx_log_t* log, bool may_install_crash_handler) {
- log_ = log;
- net_instaweb::log_message_handler::Install(log);
- if (may_install_crash_handler && install_crash_handler()) {
- NgxMessageHandler::InstallCrashHandler(log);
- }
- ngx_message_handler_->set_log(log);
- ngx_html_parse_message_handler_->set_log(log);
-}
-
-void NgxRewriteDriverFactory::SetCircularBuffer(
- SharedCircularBuffer* buffer) {
- ngx_shared_circular_buffer_ = buffer;
- ngx_message_handler_->set_buffer(buffer);
- ngx_html_parse_message_handler_->set_buffer(buffer);
-}
-
-void NgxRewriteDriverFactory::SetServerContextMessageHandler(
- ServerContext* server_context, ngx_log_t* log) {
- NgxMessageHandler* handler = new NgxMessageHandler(
- timer(), thread_system()->NewMutex());
- handler->set_log(log);
- // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
- // been raised from its default of 0.
- handler->set_buffer(ngx_shared_circular_buffer_);
- server_context_message_handlers_.insert(handler);
- defer_cleanup(new Deleter(handler));
- server_context->set_message_handler(handler);
-}
-
-void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
- // Init standard PSOL stats.
- SystemRewriteDriverFactory::InitStats(statistics);
- RewriteDriverFactory::InitStats(statistics);
- RateController::InitStats(statistics);
-
- // Init Ngx-specific stats.
- NgxServerContext::InitStats(statistics);
- InPlaceResourceRecorder::InitStats(statistics);
-}
-
-void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
- ngx_pid = ngx_getpid(); // Needed for logging to have the right PIDs.
- SystemRewriteDriverFactory::PrepareForkedProcess(name);
-}
-
-void NgxRewriteDriverFactory::NameProcess(const char* name) {
- SystemRewriteDriverFactory::NameProcess(name);
-
- // Superclass set status with prctl. Nginx has a helper function for setting
- // argv[0] as well, so let's use that. We'll show up as:
- //
- // nginx: pagespeed $name
-
- char name_for_setproctitle[32];
- snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
- "pagespeed %s", name);
- ngx_setproctitle(name_for_setproctitle);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.h b/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.h
deleted file mode 100644
index 286c6f0..0000000
--- a/lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.h
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#ifndef NGX_REWRITE_DRIVER_FACTORY_H_
-#define NGX_REWRITE_DRIVER_FACTORY_H_
-
-extern "C" {
- #include
-#if (NGX_THREADS)
- #include
-#endif
- #include
- #include
- #include
- #include
-}
-
-#include
-
-#include "pagespeed/kernel/base/md5_hasher.h"
-#include "pagespeed/kernel/base/scoped_ptr.h"
-#include "pagespeed/system/system_rewrite_driver_factory.h"
-
-namespace net_instaweb {
-
-class NgxMessageHandler;
-class NgxRewriteOptions;
-class NgxServerContext;
-class NgxUrlAsyncFetcher;
-class SharedCircularBuffer;
-class SharedMemRefererStatistics;
-class SlowWorker;
-class Statistics;
-class SystemThreadSystem;
-
-enum ProcessScriptVariablesMode {
- kOff,
- kLegacyRestricted,
- kAll
-};
-
-class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
- public:
- // We take ownership of the thread system.
- explicit NgxRewriteDriverFactory(
- const ProcessContext& process_context,
- SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
- virtual ~NgxRewriteDriverFactory();
- virtual Hasher* NewHasher();
- virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
- virtual MessageHandler* DefaultHtmlParseMessageHandler();
- virtual MessageHandler* DefaultMessageHandler();
- virtual FileSystem* DefaultFileSystem();
- virtual Timer* DefaultTimer();
- virtual NamedLockManager* DefaultLockManager();
- // Create a new RewriteOptions. In this implementation it will be an
- // NgxRewriteOptions, and it will have CoreFilters explicitly set.
- virtual RewriteOptions* NewRewriteOptions();
- virtual RewriteOptions* NewRewriteOptionsForQuery();
- virtual ServerContext* NewDecodingServerContext();
- // Check resolver configured or not.
- bool CheckResolver();
-
- // Initializes all the statistics objects created transitively by
- // NgxRewriteDriverFactory, including nginx-specific and
- // platform-independent statistics.
- static void InitStats(Statistics* statistics);
- NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
- virtual ServerContext* NewServerContext();
- virtual void ShutDown();
-
- // Starts pagespeed threads if they've not been started already. Must be
- // called after the caller has finished any forking it intends to do.
- void StartThreads();
-
- void SetServerContextMessageHandler(ServerContext* server_context,
- ngx_log_t* log);
-
- NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
-
- virtual void NonStaticInitStats(Statistics* statistics) {
- InitStats(statistics);
- }
-
- void SetMainConf(NgxRewriteOptions* main_conf);
-
- void set_resolver(ngx_resolver_t* resolver) {
- resolver_ = resolver;
- }
- void set_resolver_timeout(ngx_msec_t resolver_timeout) {
- resolver_timeout_ = resolver_timeout == NGX_CONF_UNSET_MSEC ?
- 1000 : resolver_timeout;
- }
- bool use_native_fetcher() {
- return use_native_fetcher_;
- }
- void set_use_native_fetcher(bool x) {
- use_native_fetcher_ = x;
- }
- int native_fetcher_max_keepalive_requests() {
- return native_fetcher_max_keepalive_requests_;
- }
- void set_native_fetcher_max_keepalive_requests(int x) {
- native_fetcher_max_keepalive_requests_ = x;
- }
- ProcessScriptVariablesMode process_script_variables() {
- return process_script_variables_mode_;
- }
-
- void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
-
- virtual void ShutDownMessageHandlers();
-
- virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
-
- bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
- if (!process_script_variables_set_) {
- process_script_variables_mode_ = mode;
- process_script_variables_set_ = true;
- return true;
- }
- return false;
- }
-
- virtual void PrepareForkedProcess(const char* name);
-
- virtual void NameProcess(const char* name);
-
- private:
- Timer* timer_;
-
- bool threads_started_;
- NgxMessageHandler* ngx_message_handler_;
- NgxMessageHandler* ngx_html_parse_message_handler_;
-
- std::vector ngx_url_async_fetchers_;
- ngx_log_t* log_;
- ngx_msec_t resolver_timeout_;
- ngx_resolver_t* resolver_;
- bool use_native_fetcher_;
- int native_fetcher_max_keepalive_requests_;
-
- typedef std::set NgxMessageHandlerSet;
- NgxMessageHandlerSet server_context_message_handlers_;
-
- // Owned by the superclass.
- // TODO(jefftk): merge the nginx and apache ways of doing this.
- SharedCircularBuffer* ngx_shared_circular_buffer_;
-
- GoogleString hostname_;
- int port_;
- ProcessScriptVariablesMode process_script_variables_mode_;
- bool process_script_variables_set_;
- bool shut_down_;
-
- DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
-};
-
-} // namespace net_instaweb
-
-#endif // NGX_REWRITE_DRIVER_FACTORY_H_
diff --git a/lib/ngx_pagespeed/src/ngx_rewrite_options.cc b/lib/ngx_pagespeed/src/ngx_rewrite_options.cc
deleted file mode 100644
index 290872e..0000000
--- a/lib/ngx_pagespeed/src/ngx_rewrite_options.cc
+++ /dev/null
@@ -1,555 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-#include "ngx_rewrite_options.h"
-
-extern "C" {
- #include
- #include
- #include
-}
-
-#include "ngx_pagespeed.h"
-#include "ngx_rewrite_driver_factory.h"
-
-#include "net/instaweb/public/version.h"
-#include "net/instaweb/rewriter/public/file_load_policy.h"
-#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/timer.h"
-#include "pagespeed/system/system_caches.h"
-
-namespace net_instaweb {
-
-namespace {
-
-const char kStatisticsPath[] = "StatisticsPath";
-const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
-const char kConsolePath[] = "ConsolePath";
-const char kMessagesPath[] = "MessagesPath";
-const char kAdminPath[] = "AdminPath";
-const char kGlobalAdminPath[] = "GlobalAdminPath";
-
-// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
-// indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they're entirely global and do not
-// appear in RewriteOptions. They are not alphabetized on purpose, but rather
-// left in the same order as in mod_instaweb.cc in case we end up needing to
-// compare.
-// TODO(oschaaf): this duplication is a short term solution.
-const char* const server_only_options[] = {
- "FetcherTimeoutMs",
- "FetchProxy",
- "ForceCaching",
- "GeneratedFilePrefix",
- "ImgMaxRewritesAtOnce",
- "InheritVHostConfig",
- "InstallCrashHandler",
- "MessageBufferSize",
- "NumRewriteThreads",
- "NumExpensiveRewriteThreads",
- "StaticAssetPrefix",
- "TrackOriginalContentLength",
- "UsePerVHostStatistics", // TODO(anupama): What to do about "No longer used"
- "BlockingRewriteRefererUrls",
- "CreateSharedMemoryMetadataCache",
- "LoadFromFile",
- "LoadFromFileMatch",
- "LoadFromFileRule",
- "LoadFromFileRuleMatch",
- "UseNativeFetcher",
- "NativeFetcherMaxKeepaliveRequests"
-};
-
-// Options that can only be used in the main (http) option scope.
-const char* const main_only_options[] = {
- "UseNativeFetcher",
- "NativeFetcherMaxKeepaliveRequests"
-};
-
-} // namespace
-
-RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
-
-NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
- ThreadSystem* thread_system)
- : SystemRewriteOptions(description, thread_system) {
- Init();
-}
-
-NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
- : SystemRewriteOptions(thread_system) {
- Init();
-}
-
-void NgxRewriteOptions::Init() {
- DCHECK(ngx_properties_ != NULL)
- << "Call NgxRewriteOptions::Initialize() before construction";
- clear_inherited_scripts_ = false;
- InitializeOptions(ngx_properties_);
-}
-
-void NgxRewriteOptions::AddProperties() {
- // Nginx-specific options.
- add_ngx_option(
- "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
- kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
- false);
- add_ngx_option(
- "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
- kGlobalStatisticsPath, kProcessScopeStrict,
- "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
- false);
- add_ngx_option(
- "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
- "Set the console path. Ex: /pagespeed_console", false);
- add_ngx_option(
- "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
- kServerScope, "Set the messages path. Ex: /ngx_pagespeed_message",
- false);
- add_ngx_option(
- "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
- kServerScope, "Set the admin path. Ex: /pagespeed_admin", false);
- add_ngx_option(
- "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
- kProcessScopeStrict,
- "Set the global admin path. Ex: /pagespeed_global_admin",
- false);
-
- MergeSubclassProperties(ngx_properties_);
-
- // Default properties are global but to set them the current API requires
- // a RewriteOptions instance and we're in a static method.
- NgxRewriteOptions dummy_config(NULL);
- dummy_config.set_default_x_header_value(kModPagespeedVersion);
-}
-
-void NgxRewriteOptions::Initialize() {
- if (Properties::Initialize(&ngx_properties_)) {
- SystemRewriteOptions::Initialize();
- AddProperties();
- }
-}
-
-void NgxRewriteOptions::Terminate() {
- if (Properties::Terminate(&ngx_properties_)) {
- SystemRewriteOptions::Terminate();
- }
-}
-
-bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
- StringPiece compare_directive) {
- return StringCaseEqual(config_directive, compare_directive);
-}
-
-RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
- StringPiece option_name) {
- ngx_uint_t i;
- ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
- for (i = 0; i < size; i++) {
- if (StringCaseEqual(main_only_options[i], option_name)) {
- return kProcessScopeStrict;
- }
- }
-
- size = sizeof(server_only_options) / sizeof(char*);
- for (i = 0; i < size; i++) {
- if (StringCaseEqual(server_only_options[i], option_name)) {
- return kServerScope;
- }
- }
-
- // This could be made more efficient if RewriteOptions provided a map allowing
- // access of options by their name. It's not too much of a worry at present
- // since this is just during initialization.
- for (OptionBaseVector::const_iterator it = all_options().begin();
- it != all_options().end(); ++it) {
- RewriteOptions::OptionBase* option = *it;
- if (option->option_name() == option_name) {
- // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
- // if an option is out of place.
- return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
- : option->scope();
- }
- }
- return kDirectoryScope;
-}
-
-RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
- StringPiece directive, GoogleString* msg, MessageHandler* handler) {
- EnabledEnum enabled;
- if (!ParseFromString(directive, &enabled)) {
- return RewriteOptions::kOptionNameUnknown;
- }
- if (enabled == RewriteOptions::kEnabledOff) {
- // In ngx_pagespeed, for historical reasons, we treat "off" as "unplugged".
- // Also, "off" is deprecated and people should be using "standby" or
- // "unplugged" now depending on which sense they want. See comment on
- // RewriteOptions::EnabledEnum.
- enabled = RewriteOptions::kEnabledUnplugged;
- }
- set_enabled(enabled);
- return RewriteOptions::kOptionOk;
-}
-
-RewriteOptions::OptionSettingResult
- NgxRewriteOptions::ParseAndSetOptionFromName1(
- StringPiece name, StringPiece arg,
- GoogleString* msg, MessageHandler* handler) {
- // FileCachePath needs error checking.
- if (StringCaseEqual(name, kFileCachePath)) {
- if (!StringCaseStartsWith(arg, "/")) {
- *msg = "must start with a slash";
- return RewriteOptions::kOptionValueInvalid;
- }
- }
-
- return SystemRewriteOptions::ParseAndSetOptionFromName1(
- name, arg, msg, handler);
-}
-
-template
-RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
- StringPiece option_value,
- DriverFactoryT* driver_factory,
- void (DriverFactoryT::*set_option_method)(bool)) {
- bool parsed_value;
- if (StringCaseEqual(option_value, "on") ||
- StringCaseEqual(option_value, "true")) {
- parsed_value = true;
- } else if (StringCaseEqual(option_value, "off") ||
- StringCaseEqual(option_value, "false")) {
- parsed_value = false;
- } else {
- return RewriteOptions::kOptionValueInvalid;
- }
-
- (driver_factory->*set_option_method)(parsed_value);
- return RewriteOptions::kOptionOk;
-}
-
-namespace {
-
-const char* ps_error_string_for_option(
- ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
- GoogleString msg =
- StrCat("\"", directive, "\" ", warning);
- char* s = string_piece_to_pool_string(pool, msg);
- if (s == NULL) {
- return "failed to allocate memory";
- }
- return s;
-}
-
-} // namespace
-
-// Very similar to apache/mod_instaweb::ParseDirective.
-const char* NgxRewriteOptions::ParseAndSetOptions(
- StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
- NgxRewriteDriverFactory* driver_factory,
- RewriteOptions::OptionScope scope, ngx_conf_t* cf,
- ProcessScriptVariablesMode script_mode) {
- CHECK_GE(n_args, 1);
-
- StringPiece directive = args[0];
-
- // Remove initial "ModPagespeed" if there is one.
- StringPiece mod_pagespeed("ModPagespeed");
- if (StringCaseStartsWith(directive, mod_pagespeed)) {
- directive.remove_prefix(mod_pagespeed.size());
- }
-
- if (GetOptionScope(directive) > scope) {
- return ps_error_string_for_option(
- pool, directive, "cannot be set at this scope.");
- }
-
- bool compile_scripts = false;
-
- if (script_mode != ProcessScriptVariablesMode::kOff) {
- // In the old mode we only allowed a few, so restrict to those.
- compile_scripts =
- StringCaseStartsWith(directive, "LoadFromFile") ||
- StringCaseEqual(directive, "EnableFilters") ||
- StringCaseEqual(directive, "DisableFilters") ||
- StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
- StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
- StringCaseEqual(directive,
- "DownstreamCacheRewrittenPercentageThreshold") ||
- StringCaseEqual(directive, "ShardDomain");
- // In the new behaviour we also allow scripting of query- and directory-
- // scoped options.
- compile_scripts |=
- script_mode == ProcessScriptVariablesMode::kAll &&
- (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
- (StringCaseEqual(directive, "Allow") ||
- StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
- StringCaseEqual(directive, "Disallow") ||
- StringCaseEqual(directive, "DistributableFilters") ||
- StringCaseEqual(directive, "Domain") ||
- StringCaseEqual(directive, "ExperimentVariable") ||
- StringCaseEqual(directive, "ExperimentSpec") ||
- StringCaseEqual(directive, "ForbidFilters") ||
- StringCaseEqual(directive, "RetainComment") ||
- StringCaseEqual(directive, "CustomFetchHeader") ||
- StringCaseEqual(directive, "MapOriginDomain") ||
- StringCaseEqual(directive, "MapProxyDomain") ||
- StringCaseEqual(directive, "MapRewriteDomain") ||
- StringCaseEqual(directive, "UrlValuedAttribute") ||
- StringCaseEqual(directive, "Library")));
- }
-
- ScriptLine* script_line;
- script_line = NULL;
-
- if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
- clear_inherited_scripts_ = true;
- return NGX_CONF_OK;
- }
-
- if (compile_scripts) {
- CHECK(cf != NULL);
- int i;
- // Skip the first arg which is always 'pagespeed'
- for (i = 1; i < n_args; i++) {
- ngx_str_t script_source;
-
- script_source.len = args[i].as_string().length();
- std::string tmp = args[i].as_string();
- script_source.data = reinterpret_cast(
- const_cast(tmp.c_str()));
-
- if (ngx_http_script_variables_count(&script_source) > 0) {
- ngx_http_script_compile_t* sc =
- reinterpret_cast(
- ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
- sc->cf = cf;
- sc->source = &script_source;
- sc->lengths = reinterpret_cast(
- ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
- sc->values = reinterpret_cast(
- ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
- sc->variables = 1;
- sc->complete_lengths = 1;
- sc->complete_values = 1;
- if (ngx_http_script_compile(sc) != NGX_OK) {
- return ps_error_string_for_option(
- pool, directive, "Failed to compile script variables");
- } else {
- if (script_line == NULL) {
- script_line = new ScriptLine(args, n_args, scope);
- }
- script_line->AddScriptAndArgIndex(sc, i);
- }
- }
- }
-
- if (script_line != NULL) {
- script_lines_.push_back(RefCountedPtr(script_line));
- // We have found script variables in the current configuration line, and
- // prepared the associated rewriteoptions for that.
- // We will defer parsing, validation and processing of this line to
- // request time. That means we are done handling this configuration line.
- return NGX_CONF_OK;
- }
- }
-
- GoogleString msg;
- OptionSettingResult result;
- if (n_args == 1) {
- result = ParseAndSetOptions0(directive, &msg, handler);
- } else if (n_args == 2) {
- StringPiece arg = args[1];
- if (IsDirective(directive, "UseNativeFetcher")) {
- result = ParseAndSetOptionHelper(
- arg, driver_factory,
- &NgxRewriteDriverFactory::set_use_native_fetcher);
- } else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
- int max_keepalive_requests;
- if (StringToInt(arg, &max_keepalive_requests) &&
- max_keepalive_requests > 0) {
- driver_factory->set_native_fetcher_max_keepalive_requests(
- max_keepalive_requests);
- result = RewriteOptions::kOptionOk;
- } else {
- result = RewriteOptions::kOptionValueInvalid;
- }
- } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
- if (scope == RewriteOptions::kProcessScopeStrict) {
- ProcessScriptVariablesMode mode;
- if (StringCaseEqual(arg, "all")) {
- mode = ProcessScriptVariablesMode::kAll;
- } else if (StringCaseEqual(arg, "on")) {
- mode = ProcessScriptVariablesMode::kLegacyRestricted;
- } else if (StringCaseEqual(arg, "off")) {
- mode = ProcessScriptVariablesMode::kOff;
- } else {
- return const_cast(
- "pagespeed ProcessScriptVariables: invalid value");
- }
- if (driver_factory->SetProcessScriptVariables(mode)) {
- result = RewriteOptions::kOptionOk;
- } else {
- return const_cast(
- "pagespeed ProcessScriptVariables: can only be set once");
- }
- } else {
- return const_cast(
- "ProcessScriptVariables is only allowed at the top level");
- }
- } else {
- result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
- if (result == RewriteOptions::kOptionNameUnknown) {
- result = driver_factory->ParseAndSetOption1(
- directive,
- arg,
- scope >= RewriteOptions::kLegacyProcessScope,
- &msg,
- handler);
- }
- }
- } else if (n_args == 3) {
- result = ParseAndSetOptionFromName2(directive, args[1], args[2],
- &msg, handler);
- if (result == RewriteOptions::kOptionNameUnknown) {
- result = driver_factory->ParseAndSetOption2(
- directive,
- args[1],
- args[2],
- scope >= RewriteOptions::kLegacyProcessScope,
- &msg,
- handler);
- }
- } else if (n_args == 4) {
- result = ParseAndSetOptionFromName3(
- directive, args[1], args[2], args[3], &msg, handler);
- } else {
- result = RewriteOptions::kOptionNameUnknown;
- }
-
- switch (result) {
- case RewriteOptions::kOptionOk:
- return NGX_CONF_OK;
- case RewriteOptions::kOptionNameUnknown:
- return ps_error_string_for_option(
- pool, directive, "not recognized or too many arguments");
- case RewriteOptions::kOptionValueInvalid: {
- GoogleString full_directive;
- for (int i = 0 ; i < n_args ; i++) {
- StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
- }
- return ps_error_string_for_option(pool, full_directive, msg);
- }
- }
-
- CHECK(false);
- return NULL;
-}
-
-// Execute all entries in the script_lines vector, and hand the result off to
-// ParseAndSetOptions to obtain the final option values.
-bool NgxRewriteOptions::ExecuteScriptVariables(
- ngx_http_request_t* r, MessageHandler* handler,
- NgxRewriteDriverFactory* driver_factory) {
- bool script_error = false;
-
- if (script_lines_.size() > 0) {
- std::vector >::iterator it;
- for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
- ScriptLine* script_line = it->get();
- StringPiece args[NGX_PAGESPEED_MAX_ARGS];
- std::vector::iterator cs_it;
- int i;
-
- for (i = 0; i < script_line->n_args(); i++) {
- args[i] = script_line->args()[i];
- }
-
- for (cs_it = script_line->data().begin();
- cs_it != script_line->data().end(); cs_it++) {
- ngx_http_script_compile_t* script;
- ngx_array_t* values;
- ngx_array_t* lengths;
- ngx_str_t value;
-
- script = (*cs_it)->script();
- lengths = *script->lengths;
- values = *script->values;
-
- if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
- == NULL) {
- handler->Message(kError, "ngx_http_script_run error");
- script_error = true;
- break;
- } else {
- args[(*cs_it)->index()] = str_to_string_piece(value);
- }
- }
-
- const char* status = ParseAndSetOptions(args, script_line->n_args(),
- r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
- ProcessScriptVariablesMode::kOff);
-
- if (status != NULL) {
- script_error = true;
- handler->Message(kWarning,
- "Error setting option value from script: '%s'", status);
- break;
- }
- }
- }
-
- if (script_error) {
- handler->Message(kWarning,
- "Script error(s) in configuration, disabling optimization");
- set_enabled(RewriteOptions::kEnabledOff);
- return false;
- }
-
- return true;
-}
-
-void NgxRewriteOptions::CopyScriptLinesTo(
- NgxRewriteOptions* destination) const {
- destination->script_lines_ = script_lines_;
-}
-
-void NgxRewriteOptions::AppendScriptLinesTo(
- NgxRewriteOptions* destination) const {
- destination->script_lines_.insert(destination->script_lines_.end(),
- script_lines_.begin(), script_lines_.end());
-}
-
-NgxRewriteOptions* NgxRewriteOptions::Clone() const {
- NgxRewriteOptions* options = new NgxRewriteOptions(
- StrCat("cloned from ", description()), thread_system());
- this->CopyScriptLinesTo(options);
- options->Merge(*this);
- return options;
-}
-
-const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
- const RewriteOptions* instance) {
- return dynamic_cast(instance);
-}
-
-NgxRewriteOptions* NgxRewriteOptions::DynamicCast(RewriteOptions* instance) {
- return dynamic_cast(instance);
-}
-
-} // namespace net_instaweb
diff --git a/lib/ngx_pagespeed/src/ngx_rewrite_options.h b/lib/ngx_pagespeed/src/ngx_rewrite_options.h
deleted file mode 100644
index c2274db..0000000
--- a/lib/ngx_pagespeed/src/ngx_rewrite_options.h
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Copyright 2012 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-
-// Manage configuration for pagespeed. Compare to ApacheConfig.
-
-#ifndef NGX_REWRITE_OPTIONS_H_
-#define NGX_REWRITE_OPTIONS_H_
-
-extern "C" {
- #include
- #include
- #include
-}
-
-#include
-
-#include "ngx_rewrite_driver_factory.h"
-
-#include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/ref_counted_ptr.h"
-#include "pagespeed/kernel/base/stl_util.h" // for STLDeleteElements
-#include "pagespeed/system/system_rewrite_options.h"
-
-#define NGX_PAGESPEED_MAX_ARGS 10
-
-namespace net_instaweb {
-
-class NgxRewriteDriverFactory;
-
-class ScriptArgIndex {
- public:
- explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
- : script_(script), index_(index) {
- CHECK(script != NULL);
- CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
- }
-
- virtual ~ScriptArgIndex() {}
-
- ngx_http_script_compile_t* script() { return script_; }
- int index() { return index_; }
-
- private:
- // Not owned.
- ngx_http_script_compile_t* script_;
- int index_;
-};
-
-// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
-// different rewriteoptions.
-class ScriptLine : public RefCounted {
- public:
- explicit ScriptLine(StringPiece* args, int n_args,
- RewriteOptions::OptionScope scope)
- : n_args_(n_args),
- scope_(scope) {
-
- for (int i = 0; i < n_args; i++) {
- args_[i] = args[i];
- }
- }
-
- virtual ~ScriptLine() {
- STLDeleteElements(&data_);
- data_.clear();
- }
-
- void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
- int script_index) {
- CHECK(script != NULL);
- CHECK(script_index < NGX_PAGESPEED_MAX_ARGS);
- data_.push_back(new ScriptArgIndex(script, script_index));
- }
-
- int n_args() { return n_args_;}
- StringPiece* args() { return args_;}
- RewriteOptions::OptionScope scope() { return scope_; }
- std::vector& data() {
- return data_;
- }
-
- private:
- StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
- int n_args_;
- RewriteOptions::OptionScope scope_;
- std::vector data_;
-
- DISALLOW_COPY_AND_ASSIGN(ScriptLine);
-};
-
-class NgxRewriteOptions : public SystemRewriteOptions {
- public:
- // See rewrite_options::Initialize and ::Terminate
- static void Initialize();
- static void Terminate();
-
- NgxRewriteOptions(const StringPiece& description,
- ThreadSystem* thread_system);
- explicit NgxRewriteOptions(ThreadSystem* thread_system);
- virtual ~NgxRewriteOptions() { }
-
- // args is an array of n_args StringPieces together representing a directive.
- // For example:
- // ["RewriteLevel", "PassThrough"]
- // or
- // ["EnableFilters", "combine_css,extend_cache,rewrite_images"]
- // or
- // ["ShardDomain", "example.com", "s1.example.com,s2.example.com"]
- // Apply the directive, returning NGX_CONF_OK on success or an error message
- // on failure.
- //
- // pool is a memory pool for allocating error strings.
- // cf is only required when compile_scripts is true
- // when compile_scripts is true, the rewrite_options will be prepared
- // for replacing any script $variables encountered in args. when false,
- // script variables will be substituted using the prepared rewrite options.
- const char* ParseAndSetOptions(
- StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
- NgxRewriteDriverFactory* driver_factory, OptionScope scope,
- ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
- bool ExecuteScriptVariables(
- ngx_http_request_t* r, MessageHandler* handler,
- NgxRewriteDriverFactory* driver_factory);
- void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
- void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
-
- // Make an identical copy of these options and return it.
- virtual NgxRewriteOptions* Clone() const;
-
- // Returns a suitably down cast version of 'instance' if it is an instance
- // of this class, NULL if not.
- static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
- static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
-
- const GoogleString& statistics_path() const {
- return statistics_path_.value();
- }
- const GoogleString& global_statistics_path() const {
- return global_statistics_path_.value();
- }
- const GoogleString& console_path() const {
- return console_path_.value();
- }
- const GoogleString& messages_path() const {
- return messages_path_.value();
- }
- const GoogleString& admin_path() const {
- return admin_path_.value();
- }
- const GoogleString& global_admin_path() const {
- return global_admin_path_.value();
- }
- const std::vector >& script_lines() const {
- return script_lines_;
- }
- const bool& clear_inherited_scripts() const {
- return clear_inherited_scripts_;
- }
-
- private:
- // Helper methods for ParseAndSetOptions(). Each can:
- // - return kOptionNameUnknown and not set msg:
- // - directive not handled; continue on with other possible
- // interpretations.
- // - return kOptionOk and not set msg:
- // - directive handled, all's well.
- // - return kOptionValueInvalid and set msg:
- // - directive handled with an error; return the error to the user.
- //
- // msg will be shown to the user on kOptionValueInvalid. While it would be
- // nice to always use msg and never use the MessageHandler, some option
- // parsing code in RewriteOptions expects to write to a MessageHandler. If
- // that happens we put a summary on msg so the user sees something, and the
- // detailed message goes to their log via handler.
- OptionSettingResult ParseAndSetOptions0(
- StringPiece directive, GoogleString* msg, MessageHandler* handler);
-
- virtual OptionSettingResult ParseAndSetOptionFromName1(
- StringPiece name, StringPiece arg,
- GoogleString* msg, MessageHandler* handler);
-
- // We may want to override 2- and 3-argument versions as well in the future,
- // but they are not needed yet.
-
- // Keeps the properties added by this subclass. These are merged into
- // RewriteOptions::all_properties_ during Initialize().
- //
- // RewriteOptions uses static initialization to reduce memory usage and
- // construction time. All NgxRewriteOptions instances will have the same
- // Properties, so we can build the list when we initialize the first one.
- static Properties* ngx_properties_;
- static void AddProperties();
- void Init();
-
- // Add an option to ngx_properties_
- template
- static void add_ngx_option(typename OptionClass::ValueType default_value,
- OptionClass NgxRewriteOptions::*offset,
- const char* id,
- StringPiece option_name,
- OptionScope scope,
- const char* help,
- bool safe_to_print) {
- AddProperty(default_value, offset, id, option_name, scope, help,
- safe_to_print, ngx_properties_);
- }
-
- Option statistics_path_;
- Option global_statistics_path_;
- Option console_path_;
- Option messages_path_;
- Option
and images.example.com is authorized
-// for rewriting in the config, then it needs to fetch cat.jpg from
-// images.example.com and optimize it. In apache (always) and nginx (by
-// default) we use a fetcher called "serf". This works fine, but it does run
-// its own event loop. To be more efficient, this is a "native" fetcher that
-// uses nginx's event loop.
-//
-// The fetch is started by the main thread. It will fetch the remote resource
-// from the specific url asynchronously.
-
-#ifndef NET_INSTAWEB_NGX_FETCH_H_
-#define NET_INSTAWEB_NGX_FETCH_H_
-
-extern "C" {
-#include