Initial commit

lib/ngx_pagespeed/src/log_message_handler.cc

@@ -0,0 +1,114 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jmarantz@google.com (Joshua Marantz)
+// Author: sligocki@google.com (Shawn Ligocki)
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// TODO(jefftk): share more of this code with apache's log_message_handler
+
+#include "log_message_handler.h"
+
+#include <unistd.h>
+
+#include <limits>
+#include <string>
+
+#include "base/debug/debugger.h"
+#include "base/debug/stack_trace.h"
+#include "base/logging.h"
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/string_util.h"
+
+// Make sure we don't attempt to use LOG macros here, since doing so
+// would cause us to go into an infinite log loop.
+#undef LOG
+#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION
+
+namespace {
+
+ngx_log_t* ngx_log = NULL;
+
+ngx_uint_t GetNgxLogLevel(int severity) {
+  switch (severity) {
+    case logging::LOG_INFO:
+      return NGX_LOG_INFO;
+    case logging::LOG_WARNING:
+      return NGX_LOG_WARN;
+    case logging::LOG_ERROR:
+      return NGX_LOG_ERR;
+    case logging::LOG_ERROR_REPORT:
+    case logging::LOG_FATAL:
+      return NGX_LOG_ALERT;
+    default:  // For VLOG(s)
+      return NGX_LOG_DEBUG;
+  }
+}
+
+bool LogMessageHandler(int severity, const char* file, int line,
+                       size_t message_start, const GoogleString& str) {
+  ngx_uint_t this_log_level = GetNgxLogLevel(severity);
+
+  GoogleString message = str;
+  if (severity == logging::LOG_FATAL) {
+    if (base::debug::BeingDebugged()) {
+      base::debug::BreakDebugger();
+    } else {
+      base::debug::StackTrace trace;
+      std::ostringstream stream;
+      trace.OutputToStream(&stream);
+      message.append(stream.str());
+    }
+  }
+
+  // Trim the newline off the end of the message string.
+  size_t last_msg_character_index = message.length() - 1;
+  if (message[last_msg_character_index] == '\n') {
+    message.resize(last_msg_character_index);
+  }
+
+  ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
+                net_instaweb::kModPagespeedVersion,
+                message.c_str());
+
+  if (severity == logging::LOG_FATAL) {
+    // Crash the process to generate a dump.
+    base::debug::BreakDebugger();
+  }
+
+  return true;
+}
+
+}  // namespace
+
+
+namespace net_instaweb {
+
+namespace log_message_handler {
+
+
+void Install(ngx_log_t* log_in) {
+  ngx_log = log_in;
+  logging::SetLogMessageHandler(&LogMessageHandler);
+
+  // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
+  // level is DEBUG.
+  if (ngx_log->log_level >= NGX_LOG_DEBUG) {
+    logging::SetMinLogLevel(-2);
+  }
+}
+
+}  // namespace log_message_handler
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/log_message_handler.h

@@ -0,0 +1,44 @@
+// Copyright 2010 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: jmarantz@google.com (Joshua Marantz)
+// Author: sligocki@google.com (Shawn Ligocki)
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef LOG_MESSAGE_HANDLER_H_
+#define LOG_MESSAGE_HANDLER_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+  #include <ngx_log.h>
+}
+
+namespace net_instaweb {
+
+namespace log_message_handler {
+
+// Install a log message handler that routes LOG() messages to the
+// server error log.  Should be called once at startup.  If server blocks define
+// their own logging files you would expect that LOG() messages would be routed
+// appropriately, but because logging is all done with global variables this
+// isn't possible.
+void Install(ngx_log_t* log_in);
+
+}  // namespace log_message_handler
+
+}  // namespace net_instaweb
+
+#endif  // LOG_MESSAGE_HANDLER_H_

lib/ngx_pagespeed/src/ngx_base_fetch.cc

@@ -0,0 +1,360 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
+ */
+
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
+#include <unistd.h> //for usleep
+
+#include "ngx_base_fetch.h"
+#include "ngx_event_connection.h"
+#include "ngx_list_iterator.h"
+
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/http/response_headers.h"
+
+namespace net_instaweb {
+
+const char kHeadersComplete = 'H';
+const char kFlush = 'F';
+const char kDone = 'D';
+
+NgxEventConnection* NgxBaseFetch::event_connection = NULL;
+int NgxBaseFetch::active_base_fetches = 0;
+
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
+                           NgxServerContext* server_context,
+                           const RequestContextPtr& request_ctx,
+                           PreserveCachingHeaders preserve_caching_headers,
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
+    : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
+      request_(r),
+      server_context_(server_context),
+      options_(options),
+      need_flush_(false),
+      done_called_(false),
+      last_buf_sent_(false),
+      references_(2),
+      base_fetch_type_(base_fetch_type),
+      preserve_caching_headers_(preserve_caching_headers),
+      detached_(false),
+      suppress_(false) {
+  if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
+}
+
+NgxBaseFetch::~NgxBaseFetch() {
+  pthread_mutex_destroy(&mutex_);
+  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
+}
+
+bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
+  CHECK(event_connection == NULL) << "event connection already set";
+  event_connection = new NgxEventConnection(ReadCallback);
+  return event_connection->Init(cycle);
+}
+
+void NgxBaseFetch::Terminate() {
+  if (event_connection != NULL) {
+    GoogleMessageHandler handler;
+    PosixTimer timer;
+    int64 timeout_us = Timer::kSecondUs * 30;
+    int64 end_us = timer.NowUs() + timeout_us;
+    static unsigned int sleep_microseconds = 100;
+
+    handler.Message(
+        kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
+        NgxBaseFetch::active_base_fetches);
+
+    // Try to continue processing and get the active base fetch count to 0
+    // untill the timeout expires.
+    // TODO(oschaaf): This needs more work.
+    while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
+      event_connection->Drain();
+      usleep(sleep_microseconds);
+    }
+
+    if (NgxBaseFetch::active_base_fetches != 0) {
+      handler.Message(
+          kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
+          NgxBaseFetch::active_base_fetches);
+    }
+
+    // Close down the named pipe.
+    event_connection->Shutdown();
+    delete event_connection;
+    event_connection = NULL;
+  }
+}
+
+const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
+  switch(type) {
+    case kPageSpeedResource:
+      return "ps resource";
+    case kHtmlTransform:
+      return "html transform";
+    case kAdminPage:
+      return "admin page";
+    case kIproLookup:
+      return "ipro lookup";
+    case kPageSpeedProxy:
+      return "pagespeed proxy";
+  }
+  CHECK(false);
+  return "can't get here";
+}
+
+// TODO(oschaaf): replace the ngx_log_error with VLOGS or pass in a
+// MessageHandler and use that.
+void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
+  NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
+  ngx_http_request_t* r = base_fetch->request();
+  bool detached = base_fetch->detached();
+#if (NGX_DEBUG)  // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
+  const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
+#endif
+  int refcount = base_fetch->DecrementRefCount();
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+     "pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
+     data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
+#endif
+
+  // If we ended up destructing the base fetch, or the request context is
+  // detached, skip this event.
+  if (refcount == 0 || detached) {
+    return;
+  }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  // If our request context was zeroed, skip this event.
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1081
+  if (ctx == NULL) {
+    // Should not happen normally, when it does this message will cause our
+    // system tests to fail.
+    ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+        "pagespeed [%p] skipping event: request context gone", r);
+    return;
+  }
+
+  // Normally we expect the sender to equal the active NgxBaseFetch instance.
+  DCHECK(data.sender == ctx->base_fetch);
+
+  // If someone changed our request context or NgxBaseFetch, skip processing.
+  if (data.sender != ctx->base_fetch) {
+      ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0,
+          "pagespeed [%p] skipping event: event originating from disassociated"
+          " NgxBaseFetch instance.", r);
+      return;
+  }
+
+  int rc;
+  bool run_posted = true;
+  // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
+  // we must enter a different flow. Also see ps_in_place_check_header_filter().
+  if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
+      && r->connection->error) {
+    ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+      "pagespeed [%p] request already finalized %d", r, r->count);
+    rc = NGX_ERROR;
+    run_posted = false;
+  } else {
+    rc = ps_base_fetch::ps_base_fetch_handler(r);
+  }
+
+#if (NGX_DEBUG)
+  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
+                "pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
+                r, rc, data.type);
+#endif
+
+  ngx_connection_t* c = r->connection;
+  ngx_http_finalize_request(r, rc);
+
+  if (run_posted) {
+    // See http://forum.nginx.org/read.php?2,253006,253061
+    ngx_http_run_posted_requests(c);
+  }
+}
+
+void NgxBaseFetch::Lock() {
+  pthread_mutex_lock(&mutex_);
+}
+
+void NgxBaseFetch::Unlock() {
+  pthread_mutex_unlock(&mutex_);
+}
+
+bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
+                               MessageHandler* handler) {
+  Lock();
+  buffer_.append(sp.data(), sp.size());
+  Unlock();
+  return true;
+}
+
+// should only be called in nginx thread
+ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
+  CHECK(!(done_called_ && last_buf_sent_))
+        << "CopyBufferToNginx() was called after the last buffer was sent";
+
+  // there is no buffer to send
+  if (!done_called_ && buffer_.empty()) {
+    *link_ptr = NULL;
+    return NGX_AGAIN;
+  }
+
+  int rc = string_piece_to_buffer_chain(request_->pool, buffer_, link_ptr,
+                                        done_called_ /* send_last_buf */,
+                                        need_flush_);
+  need_flush_ = false;
+
+  if (rc != NGX_OK) {
+    return rc;
+  }
+
+  // Done with buffer contents now.
+  buffer_.clear();
+
+  if (done_called_) {
+    last_buf_sent_ = true;
+    return NGX_OK;
+  }
+
+  return NGX_AGAIN;
+}
+
+// There may also be a race condition if this is called between the last Write()
+// and Done() such that we're sending an empty buffer with last_buf set, which I
+// think nginx will reject.
+ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
+  ngx_int_t rc;
+  Lock();
+  rc = CopyBufferToNginx(link_ptr);
+  Unlock();
+  return rc;
+}
+
+ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
+  const ResponseHeaders* pagespeed_headers = response_headers();
+
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }
+
+  return copy_response_headers_to_ngx(request_, *pagespeed_headers,
+                                      preserve_caching_headers_);
+}
+
+void NgxBaseFetch::RequestCollection(char type) {
+  if (suppress_) {
+    return;
+  }
+
+  // We must optimistically increment the refcount, and decrement it
+  // when we conclude we failed. If we only increment on a successfull write,
+  // there's a small chance that between writing and adding to the refcount
+  // both pagespeed and nginx will release their refcount -- destructing
+  // this NgxBaseFetch instance.
+  IncrementRefCount();
+  if (!event_connection->WriteEvent(type, this)) {
+    DecrementRefCount();
+  }
+}
+
+void NgxBaseFetch::HandleHeadersComplete() {
+  int status_code = response_headers()->status_code();
+  bool status_ok = (status_code != 0) && (status_code < 400);
+
+  if ((base_fetch_type_ != kIproLookup) || status_ok) {
+    // If this is a 404 response we need to count it in the stats.
+    if (response_headers()->status_code() == HttpStatus::kNotFound) {
+      server_context_->rewrite_stats()->resource_404_count()->Add(1);
+    }
+  }
+
+  RequestCollection(kHeadersComplete);  // Headers available.
+
+  // For the IPRO lookup, supress notification of the nginx side here.
+  // If we send both the headerscomplete event and the one from done, nasty
+  // stuff will happen if we loose the race with with the nginx side destructing
+  // this base fetch instance.
+  if (base_fetch_type_ == kIproLookup && !status_ok) {
+    suppress_ = true;
+  }
+}
+
+bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
+  Lock();
+  need_flush_ = true;
+  Unlock();
+  RequestCollection(kFlush);  // A new part of the response body is available
+  return true;
+}
+
+int NgxBaseFetch::DecrementRefCount() {
+  return DecrefAndDeleteIfUnreferenced();
+}
+
+int NgxBaseFetch::IncrementRefCount() {
+  return __sync_add_and_fetch(&references_, 1);
+}
+
+int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
+  // Creates a full memory barrier.
+  int r = __sync_add_and_fetch(&references_, -1);
+  if (r == 0) {
+    delete this;
+  }
+  return r;
+}
+
+void NgxBaseFetch::HandleDone(bool success) {
+  // TODO(jefftk): it's possible that instead of locking here we can just modify
+  // CopyBufferToNginx to only read done_called_ once.
+  CHECK(!done_called_) << "Done already called!";
+  Lock();
+  done_called_ = true;
+  Unlock();
+  RequestCollection(kDone);
+  DecrefAndDeleteIfUnreferenced();
+}
+
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_base_fetch.h

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+//
+// Collects output from pagespeed and buffers it until nginx asks for it.
+// Notifies nginx via NgxEventConnection to call ReadCallback() when
+// the headers are computed, when a flush should be performed, and when done.
+//
+//  - nginx creates a base fetch and passes it to a new proxy fetch.
+//  - The proxy fetch manages rewriting and thread complexity, and through
+//    several chained steps passes rewritten html to HandleWrite().
+//  - Written data is buffered.
+//  - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
+//    PSOL, events are written to NgxEventConnection which will end up being
+//    handled by ReadCallback() on nginx's thread.
+//    When applicable, request processing will be continued via a call to
+//    ps_base_fetch_handler().
+//  - ps_base_fetch_handler() will pull the header and body bytes from PSOL
+//    via CollectAccumulatedWrites() and write those to the module's output.
+//
+// This class is referred to in three places: the proxy fetch, nginx's request,
+// and pending events written to the associated NgxEventConnection. It must stay
+// alive until the proxy fetch and nginx request are finished, and no more
+// events are pending.
+//  - The proxy fetch will call Done() to indicate this.
+//  - nginx will call Detach() when the associated request is handled
+//    completely (e.g. the request context is about to be destroyed).
+//  - ReadCallback() will call DecrementRefCount() on instances associated to
+//    events it handles.
+//
+// When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
+
+#ifndef NGX_BASE_FETCH_H_
+#define NGX_BASE_FETCH_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include <pthread.h>
+
+#include "ngx_pagespeed.h"
+
+#include "ngx_event_connection.h"
+#include "ngx_server_context.h"
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
+
+namespace net_instaweb {
+
+enum NgxBaseFetchType {
+  kIproLookup,
+  kHtmlTransform,
+  kPageSpeedResource,
+  kAdminPage,
+  kPageSpeedProxy
+};
+
+class NgxBaseFetch : public AsyncFetch {
+ public:
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
+               NgxServerContext* server_context,
+               const RequestContextPtr& request_ctx,
+               PreserveCachingHeaders preserve_caching_headers,
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
+  virtual ~NgxBaseFetch();
+
+  // Statically initializes event_connection, require for PSOL and nginx to
+  // communicate.
+  static bool Initialize(ngx_cycle_t* cycle);
+
+  // Attempts to finish up request processing queued up in the named pipe and
+  // PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
+  // is attempted.
+  // Statically terminates and NULLS event_connection.
+  static void Terminate();
+
+  static void ReadCallback(const ps_event_data& data);
+
+  // Puts a chain in link_ptr if we have any output data buffered.  Returns
+  // NGX_OK on success, NGX_ERROR on errors.  If there's no data to send, sends
+  // data only if Done() has been called.  Indicates the end of output by
+  // setting last_buf on the last buffer in the chain.
+  //
+  // Sets link_ptr to a chain of as many buffers are needed for the output.
+  //
+  // Called by nginx in response to seeing a byte on the pipe.
+  ngx_int_t CollectAccumulatedWrites(ngx_chain_t** link_ptr);
+
+  // Copies response headers into headers_out.
+  //
+  // Called by nginx before calling CollectAccumulatedWrites() for the first
+  // time for resource fetches.  Not called at all for proxy fetches.
+  ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);
+
+  // Called by nginx to decrement the refcount.
+  int DecrementRefCount();
+
+  // Called by pagespeed to increment the refcount.
+  int IncrementRefCount();
+
+  // Detach() is called when the nginx side releases this base fetch. It
+  // sets detached_ to true and decrements the refcount. We need to know
+  // this to be able to handle events which nginx request context has been
+  // released while the event was in-flight.
+  void Detach() { detached_ = true; DecrementRefCount(); }
+
+  bool detached() { return detached_; }
+
+  ngx_http_request_t* request() { return request_; }
+  NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
+
+  bool IsCachedResultValid(const ResponseHeaders& headers) override;
+
+ private:
+  virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
+  virtual bool HandleFlush(MessageHandler* handler);
+  virtual void HandleHeadersComplete();
+  virtual void HandleDone(bool success);
+
+  // Indicate to nginx that we would like it to call
+  // CollectAccumulatedWrites().
+  void RequestCollection(char type);
+
+  // Lock must be acquired first.
+  // Returns:
+  //   NGX_ERROR: failure
+  //   NGX_AGAIN: still has buffer to send, need to checkout link_ptr
+  //   NGX_OK: done, HandleDone has been called
+  // Allocates an nginx buffer, copies our buffer_ contents into it, clears
+  // buffer_.
+  ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
+
+  void Lock();
+  void Unlock();
+
+  // Called by Done() and Release().  Decrements our reference count, and if
+  // it's zero we delete ourself.
+  int DecrefAndDeleteIfUnreferenced();
+
+  static NgxEventConnection* event_connection;
+
+  // Live count of NgxBaseFetch instances that are currently in use.
+  static int active_base_fetches;
+
+  GoogleString url_;
+  ngx_http_request_t* request_;
+  GoogleString buffer_;
+  NgxServerContext* server_context_;
+  const RewriteOptions* options_;
+  bool need_flush_;
+  bool done_called_;
+  bool last_buf_sent_;
+  // How many active references there are to this fetch. Starts at two,
+  // decremented once when Done() is called and once when Detach() is called.
+  // Incremented for each event written by pagespeed for this NgxBaseFetch, and
+  // decremented on the nginx side for each event read for it.
+  int references_;
+  pthread_mutex_t mutex_;
+  NgxBaseFetchType base_fetch_type_;
+  PreserveCachingHeaders preserve_caching_headers_;
+  // Set to true just before the nginx side releases its reference
+  bool detached_;
+  bool suppress_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_BASE_FETCH_H_

lib/ngx_pagespeed/src/ngx_caching_headers.cc

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_caching_headers.h"
+#include "ngx_list_iterator.h"
+
+#include "ngx_pagespeed.h"
+
+namespace net_instaweb {
+
+bool NgxCachingHeaders::Lookup(const StringPiece& key,
+                               StringPieceVector* values) {
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(request_->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->hash != 0 && key == str_to_string_piece(header->key)) {
+      // This will be called multiple times if there are multiple headers with
+      // this name.  Each time it will append to values.
+      SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
+                               true /* omit empty strings */);
+    }
+  }
+
+  if (values->size() == 0) {
+    // No header found with this name.
+    return false;
+  }
+
+  for (int i = 0, n = values->size(); i < n; ++i) {
+    TrimWhitespace(&((*values)[i]));
+  }
+
+  return true;
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_caching_headers.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_CACHING_HEADERS_H_
+#define NGX_CACHING_HEADERS_H_
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/http/caching_headers.h"
+
+namespace net_instaweb {
+
+// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
+// Needed so ps_header_filter can call GenerateDisabledCacheControl().
+class NgxCachingHeaders : public CachingHeaders {
+ public:
+  explicit NgxCachingHeaders(ngx_http_request_t* request)
+      : CachingHeaders(request->headers_out.status),
+        request_(request) {
+  }
+
+  virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
+
+  virtual bool IsLikelyStaticResourceType() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+  virtual bool IsCacheableResourceStatusCode() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+ private:
+  ngx_http_request_t* request_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_CACHING_HEADERS_H_

lib/ngx_pagespeed/src/ngx_event_connection.cc

@@ -0,0 +1,180 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+extern "C" {
+
+#include <ngx_channel.h>
+
+}
+
+#include "ngx_event_connection.h"
+
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/message_handler.h"
+
+namespace net_instaweb {
+
+  NgxEventConnection::NgxEventConnection(callbackPtr callback)
+    : event_handler_(callback) {
+}
+
+bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
+  int file_descriptors[2];
+
+  if (pipe(file_descriptors) != 0) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
+    return false;
+  }
+  if (ngx_nonblocking(file_descriptors[0]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[0] failed");
+  } else if (ngx_nonblocking(file_descriptors[1]) == -1) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                  ngx_nonblocking_n "pagespeed:  pipe[1] failed");
+  } else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
+    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
+                  "pagespeed: failed to create connection.");
+  } else {
+    pipe_read_fd_ = file_descriptors[0];
+    pipe_write_fd_ = file_descriptors[1];
+    // Attempt to bump the pipe capacity, because running out of buffer space
+    // can potentially lead up to writes spinning on EAGAIN.
+    // See https://github.com/pagespeed/ngx_pagespeed/issues/1380
+    // TODO(oschaaf): Consider implementing a queueing mechanism for retrying
+    // failed writes.
+#ifdef F_SETPIPE_SZ
+    fcntl(pipe_write_fd_, F_SETPIPE_SZ, 200*1024 /* minimal amount of bytes */);
+#endif
+    return true;
+  }
+  close(file_descriptors[0]);
+  close(file_descriptors[1]);
+  return false;
+}
+
+bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
+                                             ngx_fd_t pipe_fd) {
+  // pipe_fd (the read side of the pipe will end up as c->fd on the
+  // underlying ngx_connection_t that gets created here)
+  ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
+      &NgxEventConnection::ReadEventHandler);
+  return rc  == NGX_OK;
+}
+
+void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  ngx_int_t result = ngx_handle_read_event(ev, 0);
+  if (result != NGX_OK) {
+    CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
+  }
+
+  if (ev->timedout) {
+    ev->timedout = 0;
+    return;
+  }
+
+  if (!NgxEventConnection::ReadAndNotify(c->fd)) {
+    // This was copied from ngx_channel_handler(): for epoll, we need to call
+    // ngx_del_conn(). Sadly, no documentation as to why.
+    if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
+      ngx_del_conn(c, 0);
+    }
+    ngx_close_connection(c);
+    ngx_del_event(ev, NGX_READ_EVENT, 0);
+  }
+}
+
+// Deserialize ps_event_data's from the pipe as they become available.
+// Subsequently do some bookkeeping, cleanup, and error checking to keep
+// the mess out of ps_base_fetch_handler.
+bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
+  while (true) {
+    // We read only one ps_event_data at a time for now:
+    // We can end up recursing all the way and end up calling ourselves here.
+    // If that happens in the middle of looping over multiple ps_event_data's we
+    // have obtained with read(), the results from the next read() will make us
+    // process events out of order. Which can give headaches.
+    // Alternatively, we could maintain a queue to make sure we process in
+    // sequence
+    ps_event_data data;
+    ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
+
+    if (size == -1) {
+      if (errno == EINTR) {
+        continue;
+      // TODO(oschaaf): should we worry about spinning here?
+      } else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
+        return true;
+      }
+    }
+
+    if (size <= 0) {
+      return false;
+    }
+
+    data.connection->event_handler_(data);
+    return true;
+  }
+}
+
+bool NgxEventConnection::WriteEvent(void* sender) {
+  return WriteEvent('X' /* Anything char is fine */, sender);
+}
+
+bool NgxEventConnection::WriteEvent(char type, void* sender) {
+  ssize_t size = 0;
+  ps_event_data data;
+
+  ngx_memzero(&data, sizeof(data));
+  data.type = type;
+  data.sender = sender;
+  data.connection = this;
+
+  while (true) {
+    size = write(pipe_write_fd_,
+                 static_cast<void*>(&data), sizeof(data));
+    if (size == sizeof(data)) {
+      return true;
+    } else if (size == -1) {
+      // TODO(oschaaf): should we worry about spinning here?
+      if (ngx_errno == EINTR || ngx_errno == EAGAIN
+          || ngx_errno == EWOULDBLOCK) {
+        continue;
+      } else {
+        return false;
+      }
+    } else {
+      CHECK(false) << "pagespeed: unexpected return value from write(): "
+                   << size;
+    }
+  }
+  CHECK(false) << "Should not get here";
+  return false;
+}
+
+// Reads and processes what is available in the pipe.
+void NgxEventConnection::Drain() {
+  NgxEventConnection::ReadAndNotify(pipe_read_fd_);
+}
+
+void NgxEventConnection::Shutdown() {
+  close(pipe_write_fd_);
+  close(pipe_read_fd_);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_event_connection.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: oschaaf@we-amp.com (Otto van der Schaaf)
+//
+// NgxEventConnection implements a means to send events from other threads to
+// nginx's event loop, and is implemented by a named pipe under the hood.
+// A single instance is used by NgxBaseFetch, and one instance is created per
+// NgxUrlAsyncFetcher when native fetching is on.
+
+#ifndef NGX_EVENT_CONNECTION_H_
+#define NGX_EVENT_CONNECTION_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include <pthread.h>
+
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/headers.h"
+
+namespace net_instaweb {
+
+class NgxEventConnection;
+
+// Represents a single event that can be written to or read from the pipe.
+// Technically, sender is the only data we need to send. type and connection are
+// included to provide a means to trace the events along with some more
+// info.
+typedef struct {
+  char type;
+  void* sender;
+  NgxEventConnection* connection;
+} ps_event_data;
+
+// Handler signature for receiving events
+typedef void (*callbackPtr)(const ps_event_data&);
+
+// Abstracts a connection to nginx through which events can be written.
+class NgxEventConnection {
+ public:
+  explicit NgxEventConnection(callbackPtr handler);
+
+  // Creates the file descriptors and ngx_connection_t required for event
+  // messaging between pagespeed and nginx.
+  bool Init(ngx_cycle_t* cycle);
+  // Shuts down the underlying file descriptors and connection created in Init()
+  void Shutdown();
+  // Constructs a ps_event_data and writes it to the underlying named pipe.
+  bool WriteEvent(char type, void* sender);
+  // Convenience overload for clients that have a single event type.
+  bool WriteEvent(void* sender);
+  // Reads and processes what is available in the named pipe's buffer.
+  void Drain();
+ private:
+  static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
+  static void ReadEventHandler(ngx_event_t* e);
+  static bool ReadAndNotify(ngx_fd_t fd);
+
+  callbackPtr event_handler_;
+  // We own these file descriptors
+  ngx_fd_t pipe_write_fd_;
+  ngx_fd_t pipe_read_fd_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_EVENT_CONNECTION_H_

lib/ngx_pagespeed/src/ngx_fetch.cc

@@ -0,0 +1,951 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+//
+//  - The fetch is started by the main thread.
+//  - Resolver event was hooked when a NgxFetch starting. It could
+//    lookup the IP of the domain asynchronously from the DNS server.
+//  - When NgxFetchResolveDone is called, It will create the request and the
+//    connection. Add the write and read event to the epoll structure.
+//  - The read handler parses the response. Add the response to the buffer at
+//    last.
+
+// TODO(oschaaf): Currently the first applicable connection is picked from the
+// pool when re-using connections. Perhaps it would be worth it to pick the one
+// that was active the longest time ago to keep a larger pool available.
+// TODO(oschaaf): style: reindent namespace according to google C++ style guide
+// TODO(oschaaf): Retry mechanism for failures on a re-used k-a connection.
+// Currently we don't think it's going to be an issue, see the comments at
+// https://github.com/pagespeed/ngx_pagespeed/pull/781.
+
+extern "C" {
+#include <nginx.h>
+}
+
+#include "ngx_fetch.h"
+
+#include "base/logging.h"
+
+#include <algorithm>
+#include <string>
+#include <typeinfo>
+#include <vector>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/public/global_constants.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/condvar.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/pool_element.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/kernel/base/statistics.h"
+#include "pagespeed/kernel/base/string_writer.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/base/writer.h"
+#include "pagespeed/kernel/http/request_headers.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+
+namespace net_instaweb {
+
+NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
+PthreadMutex NgxConnection::connection_pool_mutex;
+// Default keepalive 60s.
+const int64 NgxConnection::keepalive_timeout_ms = 60000;
+const GoogleString NgxConnection::ka_header =
+    StrCat("keep-alive ",
+           Integer64ToString(NgxConnection::keepalive_timeout_ms));
+
+NgxConnection::NgxConnection(MessageHandler* handler,
+                             int max_keepalive_requests) {
+  c_ = NULL;
+  max_keepalive_requests_ = max_keepalive_requests;
+  handler_ = handler;
+  // max_keepalive_requests specifies the number of http requests that are
+  // allowed to be performed over a single connection. So, a
+  // max_keepalive_requests of 1 effectively disables keepalive.
+  keepalive_ = max_keepalive_requests_ > 1;
+}
+
+NgxConnection::~NgxConnection() {
+  CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
+}
+
+void NgxConnection::Terminate() {
+  for (NgxConnectionPool::iterator p = connection_pool.begin();
+       p != connection_pool.end(); ++p) {
+    NgxConnection* nc = *p;
+    ngx_close_connection(nc->c_);
+    nc->c_ = NULL;
+    delete nc;
+  }
+  connection_pool.Clear();
+}
+
+NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
+                                      MessageHandler* handler,
+                                      int max_keepalive_requests) {
+  NgxConnection* nc;
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+
+    for (NgxConnectionPool::iterator p = connection_pool.begin();
+         p != connection_pool.end(); ++p) {
+      nc = *p;
+
+      if (ngx_memn2cmp(static_cast<u_char*>(nc->sockaddr_),
+                       reinterpret_cast<u_char*>(pc->sockaddr),
+                       nc->socklen_, pc->socklen) == 0) {
+        CHECK(nc->c_->idle) << "Pool should only contain idle connections!";
+
+        nc->c_->idle = 0;
+        nc->c_->log = pc->log;
+        nc->c_->read->log = pc->log;
+        nc->c_->write->log = pc->log;
+        if (nc->c_->pool != NULL) {
+          nc->c_->pool->log = pc->log;
+        }
+
+        if (nc->c_->read->timer_set) {
+          ngx_del_timer(nc->c_->read);
+        }
+        connection_pool.Remove(nc);
+
+        ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
+                      "NgxFetch: re-using connection %p (pool size: %l)",
+                      nc, connection_pool.size());
+        return nc;
+      }
+    }
+  }
+
+  int rc = ngx_event_connect_peer(pc);
+  if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
+    return NULL;
+  }
+
+  // NgxConnection deletes itself if NgxConnection::Close()
+  nc = new NgxConnection(handler, max_keepalive_requests);
+  nc->SetSock(reinterpret_cast<u_char*>(pc->sockaddr), pc->socklen);
+  nc->c_ = pc->connection;
+  return nc;
+}
+
+void NgxConnection::Close() {
+  bool removed_from_pool = false;
+
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+    for (NgxConnectionPool::iterator p = connection_pool.begin();
+         p != connection_pool.end(); ++p) {
+      if (*p == this) {
+        // When we get here, that means that the connection either has timed
+        // out or has been closed remotely.
+        connection_pool.Remove(this);
+        ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
+                      "NgxFetch: removed connection %p (pool size: %l)",
+                      this, connection_pool.size());
+        removed_from_pool = true;
+        break;
+      }
+    }
+  }
+
+  max_keepalive_requests_--;
+
+  if (c_->read->timer_set) {
+    ngx_del_timer(c_->read);
+  }
+
+  if (c_->write->timer_set) {
+    ngx_del_timer(c_->write);
+  }
+
+  if (!keepalive_ || max_keepalive_requests_ <= 0 || removed_from_pool) {
+    ngx_close_connection(c_);
+    c_ = NULL;
+    delete this;
+    return;
+  }
+
+  ngx_add_timer(c_->read, static_cast<ngx_msec_t>(
+      NgxConnection::keepalive_timeout_ms));
+
+  c_->data = this;
+  c_->read->handler = NgxConnection::IdleReadHandler;
+  c_->write->handler = NgxConnection::IdleWriteHandler;
+  c_->idle = 1;
+
+  // This connection should not be associated with current fetch.
+  c_->log = ngx_cycle->log;
+  c_->read->log = ngx_cycle->log;
+  c_->write->log = ngx_cycle->log;
+  if (c_->pool != NULL) {
+    c_->pool->log = ngx_cycle->log;
+  }
+
+  // Allow this connection to be re-used, by adding it to the connection pool.
+  {
+    ScopedMutex lock(&NgxConnection::connection_pool_mutex);
+    connection_pool.Add(this);
+    ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
+                  "NgxFetch: Added connection %p (pool size: %l - "
+                  " max_keepalive_requests_ %d)",
+                  this, connection_pool.size(), max_keepalive_requests_);
+  }
+}
+
+void NgxConnection::IdleWriteHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  u_char buf[1];
+  int n = c->recv(c, buf, 1);
+  if (c->write->timedout) {
+    DCHECK(false) << "NgxFetch: write timeout not expected." << n;
+  }
+  if (n == NGX_AGAIN) {
+    return;
+  }
+}
+
+void NgxConnection::IdleReadHandler(ngx_event_t* ev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
+  NgxConnection* nc = static_cast<NgxConnection*>(c->data);
+
+  if (c->read->timedout) {
+    nc->set_keepalive(false);
+    nc->Close();
+    return;
+  }
+
+  char buf[1];
+  int n;
+
+  // not a timeout event, we should check connection
+  n = recv(c->fd, buf, 1, MSG_PEEK);
+  if (n == -1 && ngx_socket_errno == NGX_EAGAIN) {
+    if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
+      nc->set_keepalive(false);
+      nc->Close();
+      return;
+    }
+
+    return;
+  }
+
+  nc->set_keepalive(false);
+  nc->Close();
+}
+
+NgxFetch::NgxFetch(const GoogleString& url,
+                   AsyncFetch* async_fetch,
+                   MessageHandler* message_handler,
+                   ngx_log_t* log)
+    : str_url_(url),
+      fetcher_(NULL),
+      async_fetch_(async_fetch),
+      parser_(async_fetch->response_headers()),
+      message_handler_(message_handler),
+      bytes_received_(0),
+      fetch_start_ms_(0),
+      fetch_end_ms_(0),
+      done_(false),
+      content_length_(-1),
+      content_length_known_(false),
+      resolver_ctx_(NULL) {
+  ngx_memzero(&url_, sizeof(url_));
+  log_ = log;
+  pool_ = NULL;
+  timeout_event_ = NULL;
+  connection_ = NULL;
+}
+
+NgxFetch::~NgxFetch() {
+  if (timeout_event_ != NULL && timeout_event_->timer_set) {
+    ngx_del_timer(timeout_event_);
+  }
+  if (connection_ != NULL) {
+    connection_->Close();
+    connection_ = NULL;
+  }
+  if (pool_ != NULL) {
+    ngx_destroy_pool(pool_);
+    pool_ = NULL;
+  }
+}
+
+// This function is called by NgxUrlAsyncFetcher::StartFetch.
+bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
+  fetcher_ = fetcher;
+  bool ok = Init();
+  if (ok) {
+    ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
+                  this);
+  }  // else Init() will have emitted a reason
+  return ok;
+}
+
+// Create the pool, parse the url, add the timeout event and
+// hook the DNS resolver if needed. Else we connect directly.
+// When this returns false, our caller (NgxUrlAsyncFetcher::StartFetch)
+// will call fetch->CallbackDone()
+bool NgxFetch::Init() {
+  pool_ = ngx_create_pool(12288, log_);
+  if (pool_ == NULL) {
+    message_handler_->Message(kError, "NgxFetch: ngx_create_pool failed");
+    return false;
+  }
+
+  if (!ParseUrl()) {
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
+    return false;
+  }
+
+  timeout_event_ = static_cast<ngx_event_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_event_t)));
+  if (timeout_event_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for timeout");
+    return false;
+  }
+
+  timeout_event_->data = this;
+  timeout_event_->handler = NgxFetch::TimeoutHandler;
+  timeout_event_->log = log_;
+
+  ngx_add_timer(timeout_event_, fetcher_->fetch_timeout_);
+  r_ = static_cast<ngx_http_request_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_http_request_t)));
+
+  if (r_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for timer");
+    return false;
+  }
+  status_ = static_cast<ngx_http_status_t*>(
+      ngx_pcalloc(pool_, sizeof(ngx_http_status_t)));
+
+  if (status_ == NULL) {
+    message_handler_->Message(kError,
+                              "NgxFetch: ngx_pcalloc failed for status");
+    return false;
+  }
+
+  // The host is either a domain name or an IP address.  First check
+  // if it's a valid IP address and only if that fails fall back to
+  // using the DNS resolver.
+
+  // Maybe we have a Proxy.
+  ngx_url_t* tmp_url = &url_;
+  if (fetcher_->proxy_.url.len != 0) {
+    tmp_url = &fetcher_->proxy_;
+  }
+
+  GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
+                           tmp_url->host.len);
+  ngx_memzero(&sin_, sizeof(sin_));
+  sin_.sin_family = AF_INET;
+  sin_.sin_port = htons(tmp_url->port);
+  sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
+
+  if (sin_.sin_addr.s_addr == INADDR_NONE) {
+    // inet_addr returned INADDR_NONE, which means the hostname
+    // isn't a valid IP address.  Check DNS.
+    ngx_resolver_ctx_t temp;
+    temp.name.data = tmp_url->host.data;
+    temp.name.len = tmp_url->host.len;
+    resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
+    if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
+      // TODO(oschaaf): this spams the log, but is useful in the fetcher's
+      // current state
+      message_handler_->Message(
+          kError, "NgxFetch: Couldn't start resolving, "
+          "is there a proper resolver configured in nginx.conf?");
+      return false;
+    } else {
+      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
+                    "NgxFetch %p: start resolve for: %s",
+                    this, s_ipaddress.c_str());
+    }
+
+    resolver_ctx_->data = this;
+    resolver_ctx_->name.data = tmp_url->host.data;
+    resolver_ctx_->name.len = tmp_url->host.len;
+
+#if (nginx_version < 1005008)
+    resolver_ctx_->type = NGX_RESOLVE_A;
+#endif
+
+    resolver_ctx_->handler = NgxFetch::ResolveDoneHandler;
+    resolver_ctx_->timeout = fetcher_->resolver_timeout_;
+
+    if (ngx_resolve_name(resolver_ctx_) != NGX_OK) {
+      message_handler_->Message(kWarning,
+                                "NgxFetch: ngx_resolve_name failed");
+      return false;
+    }
+  } else {
+    if (InitRequest() != NGX_OK) {
+      message_handler()->Message(kError, "NgxFetch: InitRequest failed");
+      return false;
+    }
+  }
+  return true;
+}
+
+const char* NgxFetch::str_url() {
+  return str_url_.c_str();
+}
+
+// This function should be called only once. The only argument is sucess or
+// not.
+void NgxFetch::CallbackDone(bool success) {
+  ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
+                this, success ? "OK":"FAIL");
+
+  if (async_fetch_ == NULL) {
+    LOG(FATAL)
+        << "BUG: NgxFetch callback called more than once on same fetch"
+        << str_url_.c_str() << "(" << this << ").Please report this at"
+        << "https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss";
+    return;
+  }
+
+  release_resolver();
+
+  if (timeout_event_ && timeout_event_->timer_set) {
+    ngx_del_timer(timeout_event_);
+    timeout_event_ = NULL;
+  }
+
+  if (connection_ != NULL) {
+    // Connection will be re-used only on responses that specify
+    // 'Connection: keep-alive' in their headers.
+    bool keepalive = false;
+
+    if (success) {
+      ConstStringStarVector v;
+      if (async_fetch_->response_headers()->Lookup(
+              StringPiece(HttpAttributes::kConnection), &v)) {
+        for (size_t i = 0; i < v.size(); i++) {
+          if (*v[i] == "keep-alive") {
+            keepalive = true;
+            break;
+          } else if (*v[i] == "close") {
+            break;
+          }
+        }
+      }
+      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
+                    "NgxFetch %p: connection %p attempt keep-alive: %s",
+                    this, connection_, keepalive ? "Yes":"No");
+    }
+
+    connection_->set_keepalive(keepalive);
+    connection_->Close();
+    connection_ = NULL;
+  }
+
+  if (fetcher_ != NULL) {
+    if (fetcher_->track_original_content_length()
+        && async_fetch_->response_headers()->Has(
+            HttpAttributes::kXOriginalContentLength)) {
+      async_fetch_->extra_response_headers()->SetOriginalContentLength(
+          bytes_received_);
+    }
+    fetcher_->FetchComplete(this);
+  }
+  async_fetch_->Done(success);
+  async_fetch_ = NULL;
+}
+
+size_t NgxFetch::bytes_received() {
+  return bytes_received_;
+}
+void NgxFetch::bytes_received_add(int64 x) {
+  bytes_received_ += x;
+}
+
+int64 NgxFetch::fetch_start_ms() {
+  return fetch_start_ms_;
+}
+
+void NgxFetch::set_fetch_start_ms(int64 start_ms) {
+  fetch_start_ms_ = start_ms;
+}
+
+int64 NgxFetch::fetch_end_ms() {
+  return fetch_end_ms_;
+}
+
+void NgxFetch::set_fetch_end_ms(int64 end_ms) {
+  fetch_end_ms_ = end_ms;
+}
+
+MessageHandler* NgxFetch::message_handler() {
+  return message_handler_;
+}
+
+bool NgxFetch::ParseUrl() {
+  url_.url.len = str_url_.length();
+  url_.url.data = static_cast<u_char*>(ngx_palloc(pool_, url_.url.len));
+  if (url_.url.data == NULL) {
+    return false;
+  }
+  str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
+
+  return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
+}
+
+// Issue a request after the resolver is done
+void NgxFetch::ResolveDoneHandler(ngx_resolver_ctx_t* resolver_ctx) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
+  NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
+
+  if (resolver_ctx->state != NGX_OK) {
+    if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
+      ngx_del_timer(fetch->timeout_event());
+      fetch->set_timeout_event(NULL);
+    }
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: failed to resolve host [%.*s]", fetch,
+        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
+    fetch->CallbackDone(false);
+    return;
+  }
+
+  ngx_uint_t i;
+  // Find the first ipv4 address. We don't support ipv6 yet.
+  for (i = 0; i < resolver_ctx->naddrs; i++) {
+    // Old versions of nginx and tengine have a different definition of addrs,
+    // work around to make sure we are using the right type (ngx_addr_t*).
+    ngx_addr_t* ngx_addrs = reinterpret_cast<ngx_addr_t*>(resolver_ctx->addrs);
+    if (typeid(*ngx_addrs) == typeid(*resolver_ctx->addrs)) {
+      if (reinterpret_cast<struct sockaddr_in*>(ngx_addrs[i].sockaddr)
+              ->sin_family == AF_INET) {
+        break;
+      }
+    } else {
+      // We're using an old version that uses in_addr_t* for addrs.
+      break;
+    }
+  }
+
+  // If no suitable ipv4 address was found, we fail.
+  if (i == resolver_ctx->naddrs) {
+    if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
+      ngx_del_timer(fetch->timeout_event());
+      fetch->set_timeout_event(NULL);
+    }
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: no suitable address for host [%.*s]", fetch,
+        static_cast<int>(resolver_ctx->name.len), resolver_ctx->name.data);
+    fetch->CallbackDone(false);
+  }
+
+  ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
+
+#if (nginx_version < 1005008)
+  fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[i];
+#else
+  struct sockaddr_in* sin;
+
+  sin = reinterpret_cast<struct sockaddr_in*>(
+      resolver_ctx->addrs[i].sockaddr);
+
+  fetch->sin_.sin_family = sin->sin_family;
+  fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
+#endif
+
+  fetch->sin_.sin_family = AF_INET;
+  fetch->sin_.sin_port = htons(fetch->url_.port);
+
+  // Maybe we have Proxy
+  if (0 != fetcher->proxy_.url.len) {
+    fetch->sin_.sin_port = htons(fetcher->proxy_.port);
+  }
+
+  char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Resolved host [%V] to [%s]", fetch,
+                &resolver_ctx->name, ip_address);
+
+  fetch->release_resolver();
+
+  if (fetch->InitRequest() != NGX_OK) {
+    fetch->message_handler()->Message(kError, "NgxFetch: InitRequest failed");
+    fetch->CallbackDone(false);
+  }
+}
+
+// Prepare the request data for this fetch, and hook the write event.
+int NgxFetch::InitRequest() {
+  in_ = ngx_create_temp_buf(pool_, 4096);
+  if (in_ == NULL) {
+    return NGX_ERROR;
+  }
+
+  FixUserAgent();
+
+  RequestHeaders* request_headers = async_fetch_->request_headers();
+  ConstStringStarVector v;
+  size_t size = 0;
+  bool have_host = false;
+  GoogleString port;
+
+  response_handler = NgxFetch::HandleStatusLine;
+  int rc = Connect();
+  if (rc == NGX_AGAIN || rc == NGX_OK) {
+    if (connection_->keepalive()) {
+      request_headers->Add(HttpAttributes::kConnection,
+                           NgxConnection::ka_header);
+    }
+    const char* method = request_headers->method_string();
+    size_t method_len = strlen(method);
+
+    size = (method_len +
+            1 /* for the space */ +
+            url_.uri.len +
+            sizeof(" HTTP/1.0\r\n") - 1);
+
+    for (int i = 0; i < request_headers->NumAttributes(); i++) {
+      // if no explicit host header is given in the request headers,
+      // we need to derive it from the url.
+      if (StringCaseEqual(request_headers->Name(i), "Host")) {
+        have_host = true;
+      }
+
+      // name: value\r\n
+      size += request_headers->Name(i).length()
+          + request_headers->Value(i).length() + 4;  // 4 for ": \r\n"
+    }
+
+    if (!have_host) {
+      port = StrCat(":", IntegerToString(url_.port));
+      // for "Host: " + host + ":" + port + "\r\n"
+      size += url_.host.len + 8 + port.size();
+    }
+
+    size += 2;  // "\r\n";
+    out_ = ngx_create_temp_buf(pool_, size);
+
+    if (out_ == NULL) {
+      return NGX_ERROR;
+    }
+
+    out_->last = ngx_cpymem(out_->last, method, method_len);
+    out_->last = ngx_cpymem(out_->last, " ", 1);
+    out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
+    out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
+
+    if (!have_host) {
+      out_->last = ngx_cpymem(out_->last, "Host: ", 6);
+      out_->last = ngx_cpymem(out_->last, url_.host.data, url_.host.len);
+      out_->last = ngx_cpymem(out_->last, port.c_str(), port.size());
+      out_->last = ngx_cpymem(out_->last, "\r\n", 2);
+    }
+
+    for (int i = 0; i < request_headers->NumAttributes(); i++) {
+      const GoogleString& name = request_headers->Name(i);
+      const GoogleString& value = request_headers->Value(i);
+      out_->last = ngx_cpymem(out_->last, name.c_str(), name.length());
+      *(out_->last++) = ':';
+      *(out_->last++) = ' ';
+      out_->last = ngx_cpymem(out_->last, value.c_str(), value.length());
+      *(out_->last++) = CR;
+      *(out_->last++) = LF;
+    }
+    *(out_->last++) = CR;
+    *(out_->last++) = LF;
+    if (rc == NGX_AGAIN) {
+      return NGX_OK;
+    }
+  } else if (rc < NGX_OK) {
+    return rc;
+  }
+  CHECK(rc == NGX_OK);
+  NgxFetch::ConnectionWriteHandler(connection_->c_->write);
+  return NGX_OK;
+}
+
+int NgxFetch::Connect() {
+  ngx_peer_connection_t pc;
+  ngx_memzero(&pc, sizeof(pc));
+  pc.sockaddr = (struct sockaddr*)&sin_;
+  pc.socklen = sizeof(struct sockaddr_in);
+  pc.name = &url_.host;
+
+  // get callback is dummy function, it just returns NGX_OK
+  pc.get = ngx_event_get_peer;
+  pc.log_error = NGX_ERROR_ERR;
+  pc.log = fetcher_->log_;
+  pc.rcvbuf = -1;
+
+
+  connection_ = NgxConnection::Connect(&pc, message_handler(),
+                                       fetcher_->max_keepalive_requests_);
+  ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
+                "NgxFetch %p Connect() connection %p for [%s]",
+                this, connection_, str_url());
+
+  if (connection_ == NULL) {
+    return NGX_ERROR;
+  }
+
+  connection_->c_->write->handler = NgxFetch::ConnectionWriteHandler;
+  connection_->c_->read->handler = NgxFetch::ConnectionReadHandler;
+  connection_->c_->data = this;
+
+  // Timer set in Init() is still in effect.
+  return NGX_OK;
+}
+
+// When the fetch sends the request completely, it will hook the read event,
+// and prepare to parse the response. Timer set in Init() is still in effect.
+void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(wev->data);
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  ngx_buf_t* out = fetch->out_;
+  bool ok = true;
+  while (wev->ready && out->pos < out->last) {
+    int n = c->send(c, out->pos, out->last - out->pos);
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: ConnectionWriteHandler "
+                  "send result %d", fetch, n);
+
+    if (n >= 0) {
+      out->pos += n;
+    } else if (n == NGX_AGAIN) {
+      break;
+    } else {
+      ok = false;
+      break;
+    }
+  }
+
+  if (ok) {
+    if (out->pos == out->last) {
+      ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
+    } else {
+      ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
+    }
+  }
+
+  if (!ok) {
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch %p: failed to hook next event", fetch);
+    c->error = 1;
+    fetch->CallbackDone(false);
+  }
+}
+
+// Timer set in Init() is still in effect.
+void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
+  ngx_connection_t* c = static_cast<ngx_connection_t*>(rev->data);
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  bool ok = true;
+
+  while(rev->ready) {
+    int n = c->recv(
+        c, fetch->in_->start, fetch->in_->end - fetch->in_->start);
+
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: ConnectionReadHandler "
+                  "recv result %d", fetch, n);
+
+    if (n == NGX_AGAIN) {
+      break;
+    } else if (n == 0) {
+      // If the content length was not known, we assume that we have read
+      // all if we at least parsed the headers.
+      // If we do know the content length, having a mismatch on the bytes read
+      // will be interpreted as an error.
+      ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
+          || fetch->parser_.headers_complete();
+      fetch->done_ = true;
+      break;
+    } else if (n > 0) {
+      fetch->in_->pos = fetch->in_->start;
+      fetch->in_->last = fetch->in_->start + n;
+      ok = fetch->response_handler(c);
+      if (fetch->done_ || !ok) {
+        break;
+      }
+    }
+  }
+
+  if (!ok) {
+    fetch->CallbackDone(false);
+  } else if (fetch->done_) {
+    fetch->CallbackDone(true);
+  } else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
+    fetch->CallbackDone(false);
+  }
+}
+
+// Parse the status line: "HTTP/1.1 200 OK\r\n"
+bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle status line", fetch);
+
+  // This function only works after Nginx-1.1.4. Before nginx-1.1.4,
+  // ngx_http_parse_status_line didn't save http_version.
+  ngx_int_t n = ngx_http_parse_status_line(fetch->r_, fetch->in_,
+                                           fetch->status_);
+  if (n == NGX_ERROR) {  // parse status line error
+    fetch->message_handler()->Message(
+        kWarning, "NgxFetch: failed to parse status line");
+    return false;
+  } else if (n == NGX_AGAIN) {  // not completed
+    return true;
+  }
+
+  ResponseHeaders* response_headers =
+      fetch->async_fetch_->response_headers();
+  response_headers->SetStatusAndReason(
+      static_cast<HttpStatus::Code>(fetch->get_status_code()));
+  response_headers->set_major_version(fetch->get_major_version());
+  response_headers->set_minor_version(fetch->get_minor_version());
+
+  fetch->in_->pos += n;
+  fetch->set_response_handler(NgxFetch::HandleHeader);
+  if ((fetch->in_->last - fetch->in_->pos) > 0) {
+    return fetch->response_handler(c);
+  }
+  return true;
+}
+
+// Parse the HTTP headers
+bool NgxFetch::HandleHeader(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  char* data = reinterpret_cast<char*>(fetch->in_->pos);
+  size_t size = fetch->in_->last - fetch->in_->pos;
+  size_t n = fetch->parser_.ParseChunk(StringPiece(data, size),
+                                       fetch->message_handler_);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle headers", fetch);
+
+  if (n > size) {
+    return false;
+  } else if (fetch->parser_.headers_complete()) {
+    // TODO(oschaaf): We should also check if the request method was HEAD
+    // - but I don't think PSOL uses that at this point.
+    if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
+      fetch->done_ = true;
+    } else if (fetch->async_fetch_->response_headers()->FindContentLength(
+            &fetch->content_length_)) {
+      if (fetch->content_length_ < 0) {
+        fetch->message_handler_->Message(
+            kError, "Negative content-length in response header");
+        return false;
+      } else {
+        fetch->content_length_known_ = true;
+        if (fetch->content_length_ == 0) {
+          fetch->done_ = true;
+        }
+      }
+    }
+
+    if (fetch->fetcher_->track_original_content_length()
+        && fetch->content_length_known_) {
+      fetch->async_fetch_->response_headers()->SetOriginalContentLength(
+          fetch->content_length_);
+    }
+
+    fetch->in_->pos += n;
+    if (!fetch->done_) {
+      fetch->set_response_handler(NgxFetch::HandleBody);
+      if ((fetch->in_->last - fetch->in_->pos) > 0) {
+        return fetch->response_handler(c);
+      }
+    }
+  } else {
+    fetch->in_->pos += n;
+  }
+  return true;
+}
+
+// Read the response body
+bool NgxFetch::HandleBody(ngx_connection_t* c) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
+  char* data = reinterpret_cast<char*>(fetch->in_->pos);
+  size_t size = fetch->in_->last - fetch->in_->pos;
+
+  fetch->bytes_received_add(size);
+
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: Handle body (%d bytes)", fetch, size);
+
+  if ( fetch->async_fetch_->Write(StringPiece(data, size),
+                                  fetch->message_handler()) ) {
+    if (fetch->bytes_received_ == fetch->content_length_) {
+      fetch->done_ = true;
+    }
+    fetch->in_->pos += size;
+  } else {
+    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                  "NgxFetch %p: async fetch write failure", fetch);
+    return false;
+  }
+  return true;
+}
+
+void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
+  NgxFetch* fetch = static_cast<NgxFetch*>(tev->data);
+  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
+                "NgxFetch %p: TimeoutHandler called", fetch);
+  fetch->CallbackDone(false);
+}
+
+void NgxFetch::FixUserAgent() {
+  GoogleString user_agent;
+  ConstStringStarVector v;
+  RequestHeaders* request_headers = async_fetch_->request_headers();
+  if (request_headers->Lookup(HttpAttributes::kUserAgent, &v)) {
+    for (size_t i = 0, n = v.size(); i < n; i++) {
+      if (i != 0) {
+        user_agent += " ";
+      }
+
+      if (v[i] != NULL) {
+        user_agent += *(v[i]);
+      }
+    }
+    request_headers->RemoveAll(HttpAttributes::kUserAgent);
+  }
+  if (user_agent.empty()) {
+    user_agent += "NgxNativeFetcher";
+  }
+  GoogleString version = StrCat(
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
+  if (!StringPiece(user_agent).ends_with(version)) {
+    user_agent += version;
+  }
+  request_headers->Add(HttpAttributes::kUserAgent, user_agent);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_fetch.h

@@ -0,0 +1,210 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+//
+// PageSpeed needs some way to talk to the internet and request resources.  For
+// example, if it's optimizing www.example.com/index.html and it sees html with
+// <img src="//images.example.com/cat.jpg"> and images.example.com is authorized
+// for rewriting in the config, then it needs to fetch cat.jpg from
+// images.example.com and optimize it.  In apache (always) and nginx (by
+// default) we use a fetcher called "serf".  This works fine, but it does run
+// its own event loop.  To be more efficient, this is a "native" fetcher that
+// uses nginx's event loop.
+//
+// The fetch is started by the main thread. It will fetch the remote resource
+// from the specific url asynchronously.
+
+#ifndef NET_INSTAWEB_NGX_FETCH_H_
+#define NET_INSTAWEB_NGX_FETCH_H_
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}
+
+#include "ngx_url_async_fetcher.h"
+#include <vector>
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+#include "pagespeed/kernel/thread/pthread_mutex.h"
+
+
+namespace net_instaweb {
+
+typedef bool (*response_handler_pt)(ngx_connection_t* c);
+
+class NgxUrlAsyncFetcher;
+class NgxConnection;
+
+class NgxConnection : public PoolElement<NgxConnection> {
+ public:
+  NgxConnection(MessageHandler* handler, int max_keepalive_requests);
+  ~NgxConnection();
+  void SetSock(u_char *sockaddr, socklen_t socklen) {
+    socklen_ = socklen;
+    ngx_memcpy(&sockaddr_, sockaddr, socklen);
+  }
+  // Close ensures that NgxConnection deletes itself at the appropriate time,
+  // which can be after receiving a non-keepalive response, or when the remote
+  // server closes the connection when the NgxConnection is pooled and idle.
+  void Close();
+
+  // Once keepalive is disabled, it can't be toggled back on.
+  void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
+  bool keepalive() { return keepalive_; }
+
+  typedef Pool<NgxConnection> NgxConnectionPool;
+
+  static NgxConnection* Connect(ngx_peer_connection_t* pc,
+                                MessageHandler* handler,
+                                int max_keepalive_requests);
+  static void IdleWriteHandler(ngx_event_t* ev);
+  static void IdleReadHandler(ngx_event_t* ev);
+  // Terminate will cleanup any idle connections upon shutdown.
+  static void Terminate();
+
+  static NgxConnectionPool connection_pool;
+  static PthreadMutex connection_pool_mutex;
+
+  // c_ is owned by NgxConnection and freed in ::Close()
+  ngx_connection_t* c_;
+  static const int64 keepalive_timeout_ms;
+  static const GoogleString ka_header;
+
+ private:
+  int max_keepalive_requests_;
+  bool keepalive_;
+  socklen_t socklen_;
+  u_char sockaddr_[NGX_SOCKADDRLEN];
+  MessageHandler* handler_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxConnection);
+};
+
+class NgxFetch : public PoolElement<NgxFetch> {
+ public:
+  NgxFetch(const GoogleString& url,
+           AsyncFetch* async_fetch,
+           MessageHandler* message_handler,
+           ngx_log_t* log);
+  ~NgxFetch();
+
+  // Start the fetch.
+  bool Start(NgxUrlAsyncFetcher* fetcher);
+  // Show the completed url, for logging purposes.
+  const char* str_url();
+  // This fetch task is done. Call Done() on the async_fetch. It will copy the
+  // buffer to cache.
+  void CallbackDone(bool success);
+
+  // Show the bytes received.
+  size_t bytes_received();
+  void bytes_received_add(int64 x);
+  int64 fetch_start_ms();
+  void set_fetch_start_ms(int64 start_ms);
+  int64 fetch_end_ms();
+  void set_fetch_end_ms(int64 end_ms);
+  MessageHandler* message_handler();
+
+  int get_major_version() {
+    return static_cast<int>(status_->http_version / 1000);
+  }
+  int get_minor_version() {
+    return static_cast<int>(status_->http_version % 1000);
+  }
+  int get_status_code() {
+    return static_cast<int>(status_->code);
+  }
+  ngx_event_t* timeout_event() {
+    return timeout_event_;
+  }
+  void set_timeout_event(ngx_event_t* x) {
+    timeout_event_ = x;
+  }
+  void release_resolver() {
+    if (resolver_ctx_ != NULL && resolver_ctx_ != NGX_NO_RESOLVER) {
+      ngx_resolve_name_done(resolver_ctx_);
+      resolver_ctx_ = NULL;
+    }
+  }
+
+ private:
+  response_handler_pt response_handler;
+  // Do the initialized work and start the resolver work.
+  bool Init();
+  bool ParseUrl();
+  // Prepare the request and write it to remote server.
+  int InitRequest();
+  // Create the connection with remote server.
+  int Connect();
+  void set_response_handler(response_handler_pt handler) {
+    response_handler = handler;
+  }
+  // Only the Static functions could be used in callbacks.
+  static void ResolveDoneHandler(ngx_resolver_ctx_t* ctx);
+  // Write the request.
+  static void ConnectionWriteHandler(ngx_event_t* wev);
+  // Wait for the response.
+  static void ConnectionReadHandler(ngx_event_t* rev);
+  // Read and parse the first status line.
+  static bool HandleStatusLine(ngx_connection_t* c);
+  // Read and parse the HTTP headers.
+  static bool HandleHeader(ngx_connection_t* c);
+  // Read the response body.
+  static bool HandleBody(ngx_connection_t* c);
+  // Cancel the fetch when it's timeout.
+  static void TimeoutHandler(ngx_event_t* tev);
+
+  // Add the pagespeed User-Agent.
+  void FixUserAgent();
+  void FixHost();
+
+  const GoogleString str_url_;
+  ngx_url_t url_;
+  NgxUrlAsyncFetcher* fetcher_;
+  AsyncFetch* async_fetch_;
+  ResponseHeadersParser parser_;
+  MessageHandler* message_handler_;
+  int64 bytes_received_;
+  int64 fetch_start_ms_;
+  int64 fetch_end_ms_;
+  bool done_;
+  int64 content_length_;
+  bool content_length_known_;
+
+  struct sockaddr_in sin_;
+  ngx_log_t* log_;
+  ngx_buf_t* out_;
+  ngx_buf_t* in_;
+  ngx_pool_t* pool_;
+  ngx_http_request_t* r_;
+  ngx_http_status_t* status_;
+  ngx_event_t* timeout_event_;
+  NgxConnection* connection_;
+  ngx_resolver_ctx_t* resolver_ctx_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxFetch);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NET_INSTAWEB_NGX_FETCH_H_

lib/ngx_pagespeed/src/ngx_gzip_setter.cc

@@ -0,0 +1,403 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+#include "ngx_gzip_setter.h"
+
+#include <ngx_conf_file.h>
+
+namespace net_instaweb {
+
+NgxGZipSetter g_gzip_setter;
+
+extern "C" {
+  // These functions replace the setters for:
+  //   gzip
+  //   gzip_types
+  //   gzip_http_version
+  //   gzip_vary
+  //
+  // If these functions are called it means there is an explicit gzip
+  // configuration. The gzip configuration set by pagespeed is then rolled
+  // back and pagespeed will stop enabling gzip automatically.
+  char* ngx_gzip_redirect_conf_set_flag_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_flag_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_http_types_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_http_types_slot(cf, cmd, conf);
+    return ret;
+  }
+
+  char* ngx_gzip_redirect_conf_set_enum_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_enum_slot(cf, cmd, conf);
+    return ret;
+  }
+  char* ngx_gzip_redirect_conf_set_bitmask_slot(
+      ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    if (g_gzip_setter.enabled()) {
+      g_gzip_setter.RollBackAndDisable(cf);
+    }
+    char* ret = ngx_conf_set_bitmask_slot(cf, cmd, conf);
+    return ret;
+  }
+}
+
+NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
+NgxGZipSetter::~NgxGZipSetter() { }
+
+// Helper functions to determine signature.
+bool HasLocalConfig(ngx_command_t* command) {
+  return (!(command->type & (NGX_DIRECT_CONF|NGX_MAIN_CONF)) &&
+          command->conf == NGX_HTTP_LOC_CONF_OFFSET);
+}
+bool IsNgxFlagCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_flag_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxHttpTypesCommand(ngx_command_t* command) {
+  return (command->set == ngx_http_types_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxEnumCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_enum_slot &&
+          HasLocalConfig(command));
+}
+bool IsNgxBitmaskCommand(ngx_command_t* command) {
+  return (command->set == ngx_conf_set_bitmask_slot &&
+          HasLocalConfig(command));
+}
+
+// Initialize the NgxGzipSetter.
+// Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
+// gzip module. Enable if the signature of the zip command matches with what we
+// trust. Also sets up redirects for the configurations. These redirect handle
+// a rollback if expicit configuration is found.
+// If commands are not found the method will inform the user by logging.
+void NgxGZipSetter::Init(ngx_conf_t* cf) {
+#if (NGX_HTTP_GZIP)
+  bool gzip_signature_mismatch = false;
+  bool other_signature_mismatch = false;
+  for (int m = 0; ngx_modules[m] != NULL; m++) {
+    if (ngx_modules[m]->commands != NULL) {
+      for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
+        ngx_command_t* current_command =& ngx_modules[m]->commands[c];
+
+        // We look for the gzip command, and the exact signature we trust
+        // this means configured as an config location offset
+        // and a ngx_flag_t setter.
+        // Also see:
+        //   ngx_conf_handler in ngx_conf_file.c
+        //   ngx_http_gzip_filter_commands in ngx_http_gzip_filter.c
+        if (gzip_command_.command_ == NULL &&
+            STR_EQ_LITERAL(current_command->name, "gzip")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_command_.command_ = current_command;
+            gzip_command_.module_ = ngx_modules[m];
+            enabled_ = 1;
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip, signature mismatch");
+            gzip_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_version_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_http_version")) {
+          if (IsNgxEnumCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_enum_slot;
+            gzip_http_version_command_.command_ = current_command;
+            gzip_http_version_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_http_version, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_proxied_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_proxied")) {
+          if (IsNgxBitmaskCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_bitmask_slot;
+            gzip_proxied_command_.command_ = current_command;
+            gzip_proxied_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_proxied, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_http_types_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_types")) {
+          if (IsNgxHttpTypesCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_http_types_slot;
+            gzip_http_types_command_.command_ = current_command;
+            gzip_http_types_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_types, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+
+        if (!gzip_vary_command_.command_ &&
+            STR_EQ_LITERAL(current_command->name, "gzip_vary")) {
+          if (IsNgxFlagCommand(current_command)) {
+            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
+            gzip_vary_command_.command_ = current_command;
+            gzip_vary_command_.module_ = ngx_modules[m];
+          } else {
+            ngx_conf_log_error(
+                NGX_LOG_WARN, cf, 0,
+                "pagespeed: cannot set gzip_vary, signature mismatch");
+            other_signature_mismatch = true;
+          }
+        }
+      }
+    }
+  }
+  if (gzip_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!enabled_) {
+    // Looked through all the available commands and didn't find the "gzip" one.
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: cannot set gzip, command not found");
+    return;
+  } else if (other_signature_mismatch) {
+    return;  // Already logged error.
+  } else if (!gzip_vary_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_vary");
+    return;
+  } else if (!gzip_http_types_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_types");
+    return;
+  } else if (!gzip_http_version_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_http_version");
+    return;
+  } else if (!gzip_proxied_command_.command_) {
+    ngx_conf_log_error(
+        NGX_LOG_WARN, cf, 0, "pagespeed: missing gzip_proxied");
+    return;
+  } else {
+    return;  // Success.
+  }
+#else
+  ngx_conf_log_error(
+      NGX_LOG_WARN, cf, 0, "pagespeed: gzip not compiled into nginx");
+  return;
+#endif
+}
+
+void* ngx_command_ctx::GetConfPtr(ngx_conf_t* cf) {
+  return GetModuleConfPtr(cf) + command_->offset;
+}
+
+char* ngx_command_ctx::GetModuleConfPtr(ngx_conf_t* cf) {
+  return reinterpret_cast<char*>(
+      ngx_http_conf_get_module_loc_conf(cf, (*(module_))));
+}
+
+void NgxGZipSetter::SetNgxConfFlag(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_flag_t value) {
+  ngx_flag_t* flag = reinterpret_cast<ngx_flag_t*>(command_ctx->GetConfPtr(cf));
+  *flag = value;
+  // Save the flag position for possible rollback.
+  ngx_flags_set_.push_back(flag);
+}
+
+void NgxGZipSetter::SetNgxConfEnum(ngx_conf_t* cf,
+                                   ngx_command_ctx* command_ctx,
+                                   ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+void NgxGZipSetter::SetNgxConfBitmask(ngx_conf_t* cf,
+                                      ngx_command_ctx* command_ctx,
+                                      ngx_uint_t value) {
+  ngx_uint_t* enum_to_set =
+      reinterpret_cast<ngx_uint_t*>(command_ctx->GetConfPtr(cf));
+  *enum_to_set = value;
+  ngx_uint_set_.push_back(enum_to_set);
+}
+
+// These are the content types we want to compress.
+ngx_str_t gzip_http_types[] = {
+  ngx_string("application/ecmascript"),
+  ngx_string("application/javascript"),
+  ngx_string("application/json"),
+  ngx_string("application/pdf"),
+  ngx_string("application/postscript"),
+  ngx_string("application/x-javascript"),
+  ngx_string("image/svg+xml"),
+  ngx_string("text/css"),
+  ngx_string("text/csv"),
+  // ngx_string("text/html"),  // This is the default implied value.
+  ngx_string("text/javascript"),
+  ngx_string("text/plain"),
+  ngx_string("text/xml"),
+  ngx_null_string  // Indicates end of array.
+};
+
+gzs_enable_result NgxGZipSetter::SetGZipForLocation(ngx_conf_t* cf,
+                                                    bool value) {
+  if (!enabled_) {
+    return kEnableGZipNotEnabled;
+  }
+  if (gzip_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_command_, value);
+  }
+  return kEnableGZipOk;
+}
+
+void NgxGZipSetter::EnableGZipForLocation(ngx_conf_t* cf) {
+  if (!enabled_) {
+    return;
+  }
+
+  // When we get called twice for the same location{}, we ignore the second call
+  // to prevent adding duplicate gzip http types and so on.
+  ngx_flag_t* flag =
+      reinterpret_cast<ngx_flag_t*>(gzip_command_.GetConfPtr(cf));
+  if (*flag == 1) {
+    return;
+  }
+  SetGZipForLocation(cf, true);
+  if (gzip_vary_command_.command_) {
+    SetNgxConfFlag(cf, &gzip_vary_command_, 1);
+  }
+  if (gzip_http_version_command_.command_) {
+    SetNgxConfEnum(cf, &gzip_http_version_command_, NGX_HTTP_VERSION_10);
+  }
+  if (gzip_proxied_command_.command_) {
+    SetNgxConfBitmask(
+        cf, &gzip_http_version_command_, NGX_HTTP_GZIP_PROXIED_ANY);
+  }
+
+  // This is actually the most prone to future API changes, because gzip_types
+  // is not a simple type like ngx_flag_t. The signature check should be enough
+  // to prevent problems.
+  AddGZipHTTPTypes(cf);
+  return;
+}
+
+void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
+  if (gzip_http_types_command_.command_) {
+    // Following should not happen, but if it does return gracefully.
+    if (cf->args->nalloc < 2) {
+      ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+                         "pagespeed: unexpected small cf->args in gzip_types");
+      return;
+    }
+
+    ngx_command_t* command = gzip_http_types_command_.command_;
+    char* gzip_conf = reinterpret_cast<char* >(
+        gzip_http_types_command_.GetModuleConfPtr(cf));
+
+    // Backup the old settings.
+    ngx_str_t old_elt0 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[0];
+    ngx_str_t old_elt1 = reinterpret_cast<ngx_str_t*>(cf->args->elts)[1];
+    ngx_uint_t old_nelts = cf->args->nelts;
+
+    // Setup first arg.
+    ngx_str_t gzip_types_string = ngx_string("gzip_types");
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = gzip_types_string;
+    cf->args->nelts = 2;
+
+    ngx_str_t* http_types = gzip_http_types;
+    while (http_types->data) {
+      ngx_str_t d;
+      // We allocate the http type on the configuration pool and actually
+      // leak this if we rollback. This does not seem to be a big problem,
+      // because nginx also allocates tokens in ngx_conf_file.c and does not
+      // free them. This way they can be used safely by configurations.
+      // We must use a copy of gzip_http_types array here because nginx will
+      // manipulate the values.
+      // TODO(kspoelstra): better would be to allocate once on init and not
+      // every time we enable gzip. This needs further investigation, sharing
+      // tokens might be problematic.
+      // For now I think it is not a large problem. This might add up in case
+      // of a large multi server/location config with a lot of "pagespeed on"
+      // directives.
+      // Estimates are 300-400KB for 1000 times "pagespeed on".
+      d.data = reinterpret_cast<u_char*>(
+          ngx_pnalloc(cf->pool, http_types->len + 1));
+      snprintf(reinterpret_cast<char*>(d.data), http_types->len + 1, "%s",
+               reinterpret_cast<const char*>(http_types->data));
+      d.len = http_types->len;
+      reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = d;
+      // Call the original setter.
+      ngx_http_types_slot(cf, command, gzip_conf);
+      http_types++;
+    }
+
+    // Restore args.
+    cf->args->nelts = old_nelts;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[1] = old_elt1;
+    reinterpret_cast<ngx_str_t*>(cf->args->elts)[0] = old_elt0;
+
+    // Backup configuration location for rollback.
+    ngx_httptypes_set_.push_back(gzip_conf + command->offset);
+  }
+}
+
+void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
+  ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
+                     "pagespeed: rollback gzip, explicit configuration");
+  for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
+       i != ngx_flags_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET;
+  }
+  for (std::vector<ngx_uint_t*>::iterator i = ngx_uint_set_.begin();
+       i != ngx_uint_set_.end(); ++i) {
+    *(*i)=NGX_CONF_UNSET_UINT;
+  }
+  for (std::vector<void*>::iterator i = ngx_httptypes_set_.begin();
+       i != ngx_httptypes_set_.end(); ++i) {
+    ngx_array_t** type_array = reinterpret_cast<ngx_array_t**>(*i);
+    ngx_array_destroy(*type_array);
+    *type_array = NULL;
+  }
+  enabled_ = 0;
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_gzip_setter.h

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: kspoelstra@we-amp.com (Kees Spoelstra)
+
+/*
+ * NgxGZipSetter sets up gzip for pagespeed
+ * with the following configuration:
+ * gzip  on;
+ * gzip_vary on;
+ * gzip_types application/ecmascript;
+ * gzip_types application/javascript;
+ * gzip_types application/json;
+ * gzip_types application/pdf;
+ * gzip_types application/postscript;
+ * gzip_types application/x-javascript;
+ * gzip_types image/svg+xml;
+ * gzip_types text/css;
+ * gzip_types text/csv;
+ * gzip_types text/javascript;
+ * gzip_types text/plain;
+ * gzip_types text/xml;
+ * gzip_http_version 1.0;
+ *
+ * If there is an explicit gzip configuration in the nginx.conf
+ * pagespeed will rollback the set configuration and let the
+ * user decide what the configuration will be.
+ *
+ * It manipulates the configuration by manipulating ngx_flag_t
+ * and ngx_uint_t settings directly and using the nginx setter for
+ * gzip_http_types.
+ * This is probably a safe way to do it. If this mechanism
+ * changes all non nginx module setup & configuration will
+ * fail.
+ */
+
+#ifndef NGX_GZIP_SETTER_H_
+#define NGX_GZIP_SETTER_H_
+
+extern "C" {
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+}
+
+#include <vector>
+
+#include "ngx_pagespeed.h"
+
+#include "pagespeed/kernel/base/basictypes.h"
+
+namespace net_instaweb {
+
+// We need this class because configuration for gzip is in different modules, so
+// just saving the command will not work.
+class ngx_command_ctx {
+ public:
+  ngx_command_ctx() : command_(NULL), module_(NULL) { }
+  void* GetConfPtr(ngx_conf_t* cf);
+  char* GetModuleConfPtr(ngx_conf_t* cf);
+  ngx_command_t* command_;
+  ngx_module_t* module_;
+};
+
+enum gzs_enable_result {
+  kEnableGZipOk,
+  kEnableGZipPartial,
+  kEnableGZipNotEnabled
+};
+
+class NgxGZipSetter {
+  std::vector<ngx_flag_t*> ngx_flags_set_;
+  std::vector<ngx_uint_t*> ngx_uint_set_;
+  std::vector<void*> ngx_httptypes_set_;
+  ngx_command_ctx gzip_command_;
+  ngx_command_ctx gzip_http_types_command_;
+  ngx_command_ctx gzip_proxied_command_;
+  ngx_command_ctx gzip_vary_command_;
+  ngx_command_ctx gzip_http_version_command_;
+  bool enabled_;
+
+ public:
+  NgxGZipSetter();
+  ~NgxGZipSetter();
+  void Init(ngx_conf_t* cf);
+
+  void SetNgxConfFlag(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_flag_t value);
+  void SetNgxConfEnum(ngx_conf_t* cf,
+                      ngx_command_ctx* command_ctx,
+                      ngx_uint_t value);
+  void SetNgxConfBitmask(ngx_conf_t* cf,
+                         ngx_command_ctx* command_ctx,
+                         ngx_uint_t value);
+  void EnableGZipForLocation(ngx_conf_t* cf);
+  gzs_enable_result SetGZipForLocation(ngx_conf_t* cf, bool value);
+  void AddGZipHTTPTypes(ngx_conf_t* cf);
+  void RollBackAndDisable(ngx_conf_t* cf);
+
+  bool enabled() { return enabled_; }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(NgxGZipSetter);
+};
+
+extern NgxGZipSetter g_gzip_setter;
+
+}  // namespace net_instaweb
+
+#endif  // NGX_GZIP_SETTER_H_

lib/ngx_pagespeed/src/ngx_list_iterator.cc

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_list_iterator.h"
+
+namespace net_instaweb {
+
+NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
+    part_(part),
+    index_within_part_(0) {}
+
+ngx_table_elt_t* NgxListIterator::Next() {
+  if (index_within_part_ >= part_->nelts) {
+    if (part_->next == NULL) {
+      return NULL;
+    }
+    part_ = part_->next;
+    index_within_part_ = 0;
+  }
+  ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
+  return &elts[index_within_part_++];  // Intentional post-increment.
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_list_iterator.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+//
+// Simplifies iteration over nginx lists.
+//
+//
+
+#ifndef NGX_LIST_ITERATOR_H_
+#define NGX_LIST_ITERATOR_H_
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include "base/basictypes.h"
+
+namespace net_instaweb {
+
+class NgxListIterator {
+ public:
+  explicit NgxListIterator(ngx_list_part_t* part);
+
+  // Return the next element of the list if there is one, NULL otherwise.
+  ngx_table_elt_t* Next();
+
+ private:
+  ngx_list_part_t* part_;
+  ngx_uint_t index_within_part_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_LIST_ITERATOR_H_

lib/ngx_pagespeed/src/ngx_message_handler.cc

@@ -0,0 +1,116 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#include "ngx_message_handler.h"
+
+#include <signal.h>
+
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/abstract_mutex.h"
+#include "pagespeed/kernel/base/debug.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
+
+namespace {
+
+// This will be prefixed to every logged message.
+const char kModuleName[] = "ngx_pagespeed";
+
+// If set, the crash handler will use this to output a backtrace using
+// ngx_log_error.
+ngx_log_t* global_log = NULL;
+
+}  // namespace
+
+extern "C" {
+  static void signal_handler(int sig) {
+    // Try to output the backtrace to the log file. Since this may end up
+    // crashing/deadlocking/etc. we set an alarm() to abort us if it comes to
+    // that.
+    alarm(2);
+    if (global_log != NULL) {
+      ngx_log_error(NGX_LOG_ALERT, global_log, 0, "Trapped signal [%d]\n%s",
+                    sig, net_instaweb::StackTraceString().c_str());
+    } else {
+      fprintf(stderr, "Trapped signal [%d]\n%s\n",
+              sig, net_instaweb::StackTraceString().c_str());
+    }
+    kill(getpid(), SIGKILL);
+  }
+}  // extern "C"
+
+namespace net_instaweb {
+
+NgxMessageHandler::NgxMessageHandler(Timer* timer, AbstractMutex* mutex)
+    : SystemMessageHandler(timer, mutex),
+      log_(NULL) {
+}
+
+// Installs a signal handler for common crash signals, that tries to print
+// out a backtrace.
+void NgxMessageHandler::InstallCrashHandler(ngx_log_t* log) {
+  global_log = log;
+  signal(SIGTRAP, signal_handler);  // On check failures
+  signal(SIGABRT, signal_handler);
+  signal(SIGFPE, signal_handler);
+  signal(SIGSEGV, signal_handler);
+}
+
+ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
+  switch (type) {
+    case kInfo:
+      return NGX_LOG_INFO;
+    case kWarning:
+      return NGX_LOG_WARN;
+    case kError:
+      return NGX_LOG_ERR;
+    case kFatal:
+      return NGX_LOG_ALERT;
+  }
+
+  // This should never fall through, but some compilers seem to complain if
+  // we don't include this.
+  return NGX_LOG_ALERT;
+}
+
+void NgxMessageHandler::MessageSImpl(MessageType type,
+                                     const GoogleString& message) {
+  if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
+    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
+                  kModuleName, kModPagespeedVersion, message.c_str());
+  } else {
+    GoogleMessageHandler::MessageSImpl(type, message);
+  }
+  AddMessageToBuffer(type, message);
+}
+
+void NgxMessageHandler::FileMessageSImpl(
+    MessageType type, const char* file, int line, const GoogleString& message) {
+  if (log_ != NULL) {
+    ngx_uint_t log_level = GetNgxLogLevel(type);
+    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
+                  kModuleName, kModPagespeedVersion, file, line,
+                  message.c_str());
+  } else {
+    GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
+  }
+  AddMessageToBuffer(type, file, line, message);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_message_handler.h

@@ -0,0 +1,70 @@
+// Copyright 2013 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Author: oschaaf@gmail.com (Otto van der Schaaf)
+
+#ifndef NGX_MESSAGE_HANDLER_H_
+#define NGX_MESSAGE_HANDLER_H_
+
+extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
+  #include <ngx_core.h>
+  #include <ngx_log.h>
+}
+
+#include <cstdarg>
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/system/system_message_handler.h"
+
+namespace net_instaweb {
+
+class AbstractMutex;
+class Timer;
+
+// Implementation of a message handler that uses ngx_log_error()
+// logging to emit messages, with a fallback to GoogleMessageHandler
+class NgxMessageHandler : public SystemMessageHandler {
+ public:
+  explicit NgxMessageHandler(Timer* timer, AbstractMutex* mutex);
+
+  // Installs a signal handler for common crash signals that tries to print
+  // out a backtrace.
+  static void InstallCrashHandler(ngx_log_t* log);
+
+  void set_log(ngx_log_t* log) { log_ = log; }
+  ngx_log_t* log() { return log_; }
+
+ protected:
+  virtual void MessageSImpl(MessageType type, const GoogleString& message);
+
+  virtual void FileMessageSImpl(MessageType type, const char* file,
+                                int line, const GoogleString& message);
+
+ private:
+  ngx_uint_t GetNgxLogLevel(MessageType type);
+  ngx_log_t* log_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxMessageHandler);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_MESSAGE_HANDLER_H_

lib/ngx_pagespeed/src/ngx_pagespeed.cc

@@ -0,0 +1,3291 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+/*
+ * Usage:
+ *   server {
+ *     pagespeed    on|off;
+ *   }
+ */
+
+#include "ngx_pagespeed.h"
+
+#include <vector>
+#include <set>
+
+#include "ngx_base_fetch.h"
+#include "ngx_caching_headers.h"
+#include "ngx_gzip_setter.h"
+#include "ngx_list_iterator.h"
+#include "ngx_message_handler.h"
+#include "ngx_rewrite_driver_factory.h"
+#include "ngx_rewrite_options.h"
+#include "ngx_server_context.h"
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/cache_url_async_fetcher.h"
+#include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/public/global_constants.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/rewriter/public/experiment_matcher.h"
+#include "net/instaweb/rewriter/public/experiment_util.h"
+#include "net/instaweb/rewriter/public/process_context.h"
+#include "net/instaweb/rewriter/public/resource_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_query.h"
+#include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/util/public/fallback_property_page.h"
+#include "pagespeed/automatic/proxy_fetch.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/null_message_handler.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/stack_buffer.h"
+#include "pagespeed/kernel/base/stdio_file_system.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_writer.h"
+#include "pagespeed/kernel/base/time_util.h"
+#include "pagespeed/kernel/http/content_type.h"
+#include "pagespeed/kernel/http/google_url.h"
+#include "pagespeed/kernel/http/query_params.h"
+#include "pagespeed/kernel/html/html_keywords.h"
+#include "pagespeed/kernel/thread/pthread_shared_mem.h"
+#include "pagespeed/kernel/util/gzip_inflater.h"
+#include "pagespeed/kernel/util/statistics_logger.h"
+#include "pagespeed/system/in_place_resource_recorder.h"
+#include "pagespeed/system/system_caches.h"
+#include "pagespeed/system/system_request_context.h"
+#include "pagespeed/system/system_rewrite_options.h"
+#include "pagespeed/system/system_server_context.h"
+#include "pagespeed/system/system_thread_system.h"
+
+extern ngx_module_t ngx_pagespeed;
+
+// Unused flag, see
+// http://lxr.evanmiller.org/http/source/http/ngx_http_request.h#L130
+#define  NGX_HTTP_PAGESPEED_BUFFERED 0x08
+#define  POST_BUF_READ_SIZE 65536
+
+// Needed for SystemRewriteDriverFactory to use shared memory.
+#define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
+#define NGINX_1_13_4 1013004
+
+net_instaweb::NgxRewriteDriverFactory* active_driver_factory = NULL;
+
+namespace net_instaweb {
+
+const char* kInternalEtagName = "@psol-etag";
+// The process context takes care of proactively initialising
+// a few libraries for us, some of which are not thread-safe
+// when they are initialized lazily.
+ProcessContext* process_context = new ProcessContext();
+bool process_context_cleanup_hooked = false;
+
+StringPiece str_to_string_piece(ngx_str_t s) {
+  return StringPiece(reinterpret_cast<char*>(s.data), s.len);
+}
+
+// Nginx uses memory pools, like Apache, allocating strings a request needs from
+// the pool and then throwing it all away when the request finishes.  Use this
+// when you need to pass a short string to nginx and want it to take ownership
+// of the string.
+char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp) {
+  // Need space for the final null.
+  ngx_uint_t buffer_size = sp.size() + 1;
+  char* s = static_cast<char*>(ngx_palloc(pool, buffer_size));
+  if (s == NULL) {
+    LOG(ERROR) << "string_piece_to_pool_string: ngx_palloc() returned NULL";
+    DCHECK(false);
+    return NULL;
+  }
+  sp.copy(s, buffer_size /* max to copy */);
+  s[buffer_size-1] = '\0';  // Null terminate it.
+  return s;
+}
+
+// When passing the body of http responses between filters Nginx uses a linked
+// list of buffers ("buffer chain"), again like Apache.  This constructs one of
+// those lists from a StringPiece.  This is what you use when you need to pass a
+// (potentially) longer string to nginx and want it to take ownership.
+ngx_int_t string_piece_to_buffer_chain(
+    ngx_pool_t* pool, StringPiece sp, ngx_chain_t** link_ptr,
+    bool send_last_buf, bool send_flush) {
+  // Below, *link_ptr will be NULL if we're starting the chain, and the head
+  // chain link.
+  *link_ptr = NULL;
+
+  // If non-null, the current last link in the chain.
+  ngx_chain_t* tail_link = NULL;
+
+  // How far into sp we're currently working on.
+  ngx_uint_t offset;
+
+  // Other modules seem to default to ngx_pagesize.
+  ngx_uint_t max_buffer_size = ngx_pagesize;
+  for (offset = 0 ;
+       offset < sp.size() ||
+           // If we need to send the last buffer bit and there's no data, we
+           // should send a single empty buffer.  Otherwise we shouldn't
+           // generate empty buffers.
+           (offset == 0 && sp.size() == 0);
+       offset += max_buffer_size) {
+    // Prepare a new nginx buffer to put our buffered writes into.
+    ngx_buf_t* b = static_cast<ngx_buf_t*>(ngx_calloc_buf(pool));
+    if (b == NULL) {
+      return NGX_ERROR;
+    }
+
+    if (sp.size() == 0) {
+      CHECK(offset == 0);                                          // NOLINT
+      b->pos = b->start = b->end = b->last = NULL;
+      // The purpose of this buffer is just to pass along last_buf.
+      b->sync = 1;
+    } else {
+      CHECK(sp.size() > offset);
+      ngx_uint_t b_size = sp.size() - offset;
+      if (b_size > max_buffer_size) {
+        b_size = max_buffer_size;
+      }
+
+      b->start = b->pos = static_cast<u_char*>(ngx_palloc(pool, b_size));
+      if (b->pos == NULL) {
+        return NGX_ERROR;
+      }
+
+      // Copy our writes over.  We're copying from sp[offset] up to
+      // sp[offset + b_size] into b which has size b_size.
+      sp.copy(reinterpret_cast<char*>(b->pos), b_size, offset);
+      b->last = b->end = b->pos + b_size;
+
+      b->temporary = 1;  // Identify this buffer as in-memory and mutable.
+    }
+
+    // Prepare a chain link.
+    ngx_chain_t* cl = static_cast<ngx_chain_t*>(ngx_alloc_chain_link(pool));
+    if (cl == NULL) {
+      return NGX_ERROR;
+    }
+
+    cl->buf = b;
+    cl->next = NULL;
+
+    if (*link_ptr == NULL) {
+      // This is the first link in the returned chain.
+      *link_ptr = cl;
+    } else {
+      // Link us into the chain.
+      CHECK(tail_link != NULL);
+      tail_link->next = cl;
+    }
+
+    tail_link = cl;
+  }
+
+
+  CHECK(tail_link != NULL);
+  if (send_flush) {
+    tail_link->buf->flush = true;
+  }
+  if (send_last_buf) {
+    tail_link->buf->last_buf = true;
+  }
+
+  return NGX_OK;
+}
+
+// Get the context for this request.  ps_connection_read_handler should already
+// have been called to create it.
+ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r) {
+  return static_cast<ps_request_ctx_t*>(
+      ngx_http_get_module_ctx(r, ngx_pagespeed));
+}
+
+// Tell nginx whether we have network activity we're waiting for so that it sets
+// a write handler.  See src/http/ngx_http_request.c:2083.
+void ps_set_buffered(ngx_http_request_t* r, bool on) {
+  if (on) {
+    r->buffered |= NGX_HTTP_PAGESPEED_BUFFERED;
+  } else {
+    r->buffered &= ~NGX_HTTP_PAGESPEED_BUFFERED;
+  }
+}
+
+namespace {
+
+void ps_release_base_fetch(ps_request_ctx_t* ctx);
+
+}  // namespace
+
+namespace ps_base_fetch {
+
+ngx_http_output_header_filter_pt ngx_http_next_header_filter;
+ngx_http_output_body_filter_pt ngx_http_next_body_filter;
+
+ngx_int_t ps_base_fetch_filter(ngx_http_request_t* r, ngx_chain_t* in) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  if (r->header_only) {
+    return NGX_OK;
+  }
+  if (ctx == NULL || ctx->base_fetch == NULL) {
+    return ngx_http_next_body_filter(r, in);
+  }
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "http pagespeed write filter \"%V\"", &r->uri);
+
+  // send response body
+  if (in || r->connection->buffered) {
+    ngx_int_t rc = ngx_http_next_body_filter(r, in);
+    // We can't indicate that we are done yet, because we have an active base
+    // fetch associated to this request.
+    if (rc != NGX_OK) {
+      return rc;
+    }
+  }
+
+  return NGX_AGAIN;
+}
+
+// This runs on the nginx event loop in response to seeing the byte PageSpeed
+// sent over the pipe to trigger the nginx-side code.  Copy whatever is ready
+// from PageSpeed out to the browser (headers and/or body).
+ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+  ngx_int_t rc;
+  ngx_chain_t* cl = NULL;
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "ps fetch handler: %V", &r->uri);
+
+  if (ngx_terminate || ngx_exiting) {
+    ps_set_buffered(r, false);
+    ps_release_base_fetch(ctx);
+    return NGX_ERROR;
+  }
+
+  if (!r->header_sent) {
+    int status_code = ctx->base_fetch->response_headers()->status_code();
+    bool status_ok = (status_code != 0) && (status_code < 400);
+
+    // Pass on error handling for non-success status codes to nginx for
+    // responses where PSOL acts as a content generator (pagespeed resource,
+    // ipro hit, admin pages).
+    // This generates nginx's default error responses, but also allows header
+    // modules running after us to manipulate those responses.
+    if (!status_ok && (ctx->base_fetch->base_fetch_type() != kHtmlTransform
+                       && ctx->base_fetch->base_fetch_type() != kIproLookup)) {
+      ps_release_base_fetch(ctx);
+      ngx_http_filter_finalize_request(r, NULL, status_code);
+      return NGX_DONE;
+    }
+
+    if (ctx->preserve_caching_headers != kDontPreserveHeaders) {
+      ngx_table_elt_t* header;
+      NgxListIterator it(&(r->headers_out.headers.part));
+      while ((header = it.Next()) != NULL) {
+        // We need to remember a few headers when ModifyCachingHeaders is off,
+        // so we can send them unmodified in copy_response_headers_to_ngx().
+        // This just sets the hash to 0 for all other headers. That way, we
+        // avoid  some relatively complicated code to reconstruct these headers.
+        if (!(STR_CASE_EQ_LITERAL(header->key, "Cache-Control") ||
+              (ctx->preserve_caching_headers == kPreserveAllCachingHeaders &&
+               (STR_CASE_EQ_LITERAL(header->key, "Etag") ||
+                STR_CASE_EQ_LITERAL(header->key, "Date") ||
+                STR_CASE_EQ_LITERAL(header->key, "Last-Modified") ||
+                STR_CASE_EQ_LITERAL(header->key, "Expires"))))) {
+          header->hash = 0;
+          if (STR_CASE_EQ_LITERAL(header->key, "Location")) {
+            // There's a possible issue with the location header, where setting
+            // the hash to 0 is not enough. See:
+            // https://github.com/nginx/nginx/blob/master/src/http/ngx_http_header_filter_module.c#L314
+            r->headers_out.location = NULL;
+          }
+        }
+      }
+    } else {
+      ngx_http_clean_header(r);
+    }
+    // collect response headers from pagespeed
+    rc = ctx->base_fetch->CollectHeaders(&r->headers_out);
+    if (rc == NGX_ERROR) {
+      ps_release_base_fetch(ctx);
+      return NGX_HTTP_INTERNAL_SERVER_ERROR;
+    }
+
+    // send response headers
+    rc = ngx_http_next_header_filter(r);
+
+    // standard nginx send header check see ngx_http_send_response
+    if (rc == NGX_ERROR || rc > NGX_OK) {
+      ps_release_base_fetch(ctx);
+      return ngx_http_filter_finalize_request(r, NULL, rc);
+    }
+
+    // for in_place_check_header_filter
+    if (rc < NGX_OK && rc != NGX_AGAIN) {
+      CHECK(rc == NGX_DONE);
+      return NGX_DONE;
+    }
+
+    ps_set_buffered(r, true);
+  }
+
+  // collect response body from pagespeed
+  // Pass the optimized content along to later body filters.
+  // From Weibin: This function should be called mutiple times. Store the
+  // whole file in one chain buffers is too aggressive. It could consume
+  // too much memory in busy servers.
+
+  rc = ctx->base_fetch->CollectAccumulatedWrites(&cl);
+  ngx_log_error(NGX_LOG_DEBUG, ctx->r->connection->log, 0,
+                "CollectAccumulatedWrites, %d", rc);
+
+  if (rc == NGX_ERROR) {
+    ps_set_buffered(r, false);
+    ps_release_base_fetch(ctx);
+    return NGX_HTTP_INTERNAL_SERVER_ERROR;
+  }
+
+  if (rc == NGX_AGAIN && cl == NULL) {
+    // there is no body buffer to send now.
+    return NGX_AGAIN;
+  }
+
+  if (rc == NGX_OK) {
+    ps_set_buffered(r, false);
+    ps_release_base_fetch(ctx);
+  }
+
+  return ps_base_fetch_filter(r, cl);
+}
+
+void ps_base_fetch_filter_init() {
+  ngx_http_next_header_filter = ngx_http_top_header_filter;
+  ngx_http_next_body_filter = ngx_http_top_body_filter;
+  ngx_http_top_body_filter = ps_base_fetch_filter;
+}
+
+}  // namespace ps_base_fetch
+
+
+namespace {
+
+// Setting headers in nginx is tricky because it's not just a matter of adding
+// them to a list.  You also need to remove them if there's already one there,
+// as well as setting the shortcut pointers (both upper case and lower case).
+//
+// Based on ngx_http_add_cache_control.
+ngx_int_t ps_set_cache_control(ngx_http_request_t* r, char* cache_control) {
+  // First strip existing cache-control headers.
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "Cache-Control")) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+
+  // Now add our new cache control header.
+  if (r->headers_out.cache_control.elts == NULL) {
+    ngx_int_t rc = ngx_array_init(&r->headers_out.cache_control, r->pool,
+                                  1, sizeof(ngx_table_elt_t*));
+    if (rc != NGX_OK) {
+      return NGX_ERROR;
+    }
+  }
+  ngx_table_elt_t** cache_control_headers = static_cast<ngx_table_elt_t**>(
+      ngx_array_push(&r->headers_out.cache_control));
+  if (cache_control_headers == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0] = static_cast<ngx_table_elt_t*>(
+      ngx_list_push(&r->headers_out.headers));
+  if (cache_control_headers[0] == NULL) {
+    return NGX_ERROR;
+  }
+  cache_control_headers[0]->hash = 1;
+  ngx_str_set(&cache_control_headers[0]->key, "Cache-Control");
+  cache_control_headers[0]->value.len = strlen(cache_control);
+  cache_control_headers[0]->value.data =
+      reinterpret_cast<u_char*>(cache_control);
+
+  return NGX_OK;
+}
+
+// Returns false if the header wasn't found.  Otherwise sets cache_control and
+// returns true;
+bool ps_get_cache_control(ngx_http_request_t* r, GoogleString* cache_control) {
+  // Use headers_out.cache_control instead of looking for Cache-Control in
+  // headers_out.headers, because if an upstream sent multiple Cache-Control
+  // headers they're already combined in headers_out.cache_control.
+  auto ccp = static_cast<ngx_table_elt_t**>(r->headers_out.cache_control.elts);
+  if (ccp == nullptr) {
+    return false;  // Header not present.
+  }
+  bool first_segment = true;
+  for (ngx_uint_t i = 0; i < r->headers_out.cache_control.nelts; i++) {
+    if (ccp[i]->hash == 0) {
+      continue;  // Elements with a hash of 0 are marked as excluded.
+    }
+    if (first_segment) {
+      first_segment = false;
+    } else {
+      cache_control->append(", ");
+    }
+    cache_control->append(reinterpret_cast<char*>(ccp[i]->value.data),
+                          ccp[i]->value.len);
+  }
+  return true;
+}
+
+template<class Headers>
+void copy_headers_from_table(const ngx_list_t &from, Headers* to) {
+  // Standard nginx idiom for iterating over a list.  See ngx_list.h
+  ngx_uint_t i;
+  const ngx_list_part_t* part = &from.part;
+  const ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
+
+  for (i = 0 ; /* void */; i++) {
+    if (i >= part->nelts) {
+      if (part->next == NULL) {
+        break;
+      }
+
+      part = part->next;
+      header = static_cast<ngx_table_elt_t*>(part->elts);
+      i = 0;
+    }
+    // Make sure we don't copy over headers that are unset.
+    if (header[i].hash == 0) {
+      continue;
+    }
+    StringPiece key = str_to_string_piece(header[i].key);
+    StringPiece value = str_to_string_piece(header[i].value);
+
+    to->Add(key, value);
+  }
+}
+}  // namespace
+
+void copy_response_headers_from_ngx(const ngx_http_request_t* r,
+                                    ResponseHeaders* headers) {
+  headers->set_major_version(r->http_version / 1000);
+  headers->set_minor_version(r->http_version % 1000);
+  copy_headers_from_table(r->headers_out.headers, headers);
+
+  headers->set_status_code(r->headers_out.status);
+
+  // Manually copy over the content type because it's not included in
+  // request_->headers_out.headers.
+  headers->Add(HttpAttributes::kContentType,
+               str_to_string_piece(r->headers_out.content_type));
+
+  // When we don't have a date header, set one with the current time.
+  if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
+    PosixTimer timer;
+    headers->SetDate(timer.NowMs());
+  }
+
+  // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
+  headers->ComputeCaching();
+}
+
+void copy_request_headers_from_ngx(const ngx_http_request_t* r,
+                                   RequestHeaders* headers) {
+  // TODO(chaizhenhua): only allow RewriteDriver::kPassThroughRequestAttributes?
+  headers->set_major_version(r->http_version / 1000);
+  headers->set_minor_version(r->http_version % 1000);
+  copy_headers_from_table(r->headers_in.headers, headers);
+}
+
+// PSOL produces caching headers that need some changes before we can send them
+// out.  Make those changes and populate r->headers_out from pagespeed_headers.
+ngx_int_t copy_response_headers_to_ngx(
+    ngx_http_request_t* r,
+    const ResponseHeaders& pagespeed_headers,
+    PreserveCachingHeaders preserve_caching_headers) {
+  ngx_http_headers_out_t* headers_out = &r->headers_out;
+  headers_out->status = pagespeed_headers.status_code();
+
+  ngx_int_t i;
+  for (i = 0 ; i < pagespeed_headers.NumAttributes() ; i++) {
+    // For IPRO cache misses, these gs_ variables may point to freed memory
+    // when nginx writes the headers to the output as the NgxBaseFetch instance
+    // that owns this memory gets released during request processing. So we
+    // copy these strings later on.
+    const GoogleString& name_gs = pagespeed_headers.Name(i);
+    const GoogleString& value_gs = pagespeed_headers.Value(i);
+
+    if (preserve_caching_headers == kPreserveAllCachingHeaders) {
+      if (StringCaseEqual(name_gs, "ETag") ||
+          StringCaseEqual(name_gs, "Expires") ||
+          StringCaseEqual(name_gs, "Date") ||
+          StringCaseEqual(name_gs, "Last-Modified") ||
+          StringCaseEqual(name_gs, "Cache-Control")) {
+        continue;
+      }
+    } else if (preserve_caching_headers == kPreserveOnlyCacheControl) {
+      // Retain the original Cache-Control header, but send the recomputed
+      // values for all other cache-related headers.
+      if (StringCaseEqual(name_gs, "Cache-Control")) {
+        continue;
+      }
+    }  // else we don't preserve any headers.
+
+    ngx_str_t name, value;
+    value.len = value_gs.size();
+    value.data = reinterpret_cast<u_char*>(
+        string_piece_to_pool_string(r->pool, value_gs.c_str()));
+
+    // To prevent the gzip module from clearing weak etags, we output them
+    // using a different name here. The etag header filter module runs behind
+    // the gzip compressors header filter, and will rename it to 'ETag'
+    if (StringCaseEqual(name_gs, "etag")
+        && StringCaseStartsWith(value_gs, "W/")) {
+      name.len = strlen(kInternalEtagName);
+      name.data = reinterpret_cast<u_char*>(
+          const_cast<char*>(kInternalEtagName));
+    } else {
+      name.len = name_gs.size();
+      name.data = reinterpret_cast<u_char*>(
+          string_piece_to_pool_string(r->pool, name_gs.c_str()));
+    }
+
+    // In case string_piece_to_pool_string failed:
+    if (name.data == NULL || value.data == NULL) {
+        return NGX_ERROR;
+    }
+
+    // TODO(jefftk): If we're setting a cache control header we'd like to
+    // prevent any downstream code from changing it.  Specifically, if we're
+    // serving a cache-extended resource the url will change if the resource
+    // does and so we've given it a long lifetime.  If the site owner has done
+    // something like set all css files to a 10-minute cache lifetime, that
+    // shouldn't apply to our generated resources.  See Apache code in
+    // net/instaweb/apache/header_util:AddResponseHeadersToRequest
+
+    // Make copies of name and value to put into headers_out.
+    if (STR_EQ_LITERAL(name, "Cache-Control")) {
+      ps_set_cache_control(r, reinterpret_cast<char*>(value.data));
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Content-Type")) {
+      // Unlike all the other headers, content_type is just a string.
+      headers_out->content_type = value;
+
+      // We should not include the charset when determining content_type_len, so
+      // scan for the ';' that marks the start of the charset part.
+      for (ngx_uint_t i = 0; i < value.len; i++) {
+        if (value.data[i] == ';') {
+          break;
+        }
+        headers_out->content_type_len = i + 1;
+      }
+
+      // In ngx_http_test_content_type() nginx will allocate and calculate
+      // content_type_lowcase if we leave it as null.
+      headers_out->content_type_lowcase = NULL;
+      continue;
+      // TODO(oschaaf): are there any other headers we should not try to
+      // copy here?
+    } else if (STR_EQ_LITERAL(name, "Connection")) {
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Keep-Alive")) {
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
+      continue;
+    } else if (STR_EQ_LITERAL(name, "Vary") && value.len
+        && STR_EQ_LITERAL(value, "Accept-Encoding")) {
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      ctx->psol_vary_accept_only = true;
+    }
+
+    ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
+        ngx_list_push(&headers_out->headers));
+    if (header == NULL) {
+      return NGX_ERROR;
+    }
+
+    header->hash = 1;  // Include this header in the output.
+    header->key.data = name.data;
+    header->key.len = name.len;
+    header->value.data = value.data;
+    header->value.len = value.len;
+
+    // Populate the shortcuts to commonly used headers.
+    if (STR_EQ_LITERAL(name, "Date")) {
+      headers_out->date = header;
+    } else if (STR_EQ_LITERAL(name, "Etag")) {
+      headers_out->etag = header;
+    } else if (STR_EQ_LITERAL(name, "Expires")) {
+      headers_out->expires = header;
+    } else if (STR_EQ_LITERAL(name, "Last-Modified")) {
+      headers_out->last_modified = header;
+    } else if (STR_EQ_LITERAL(name, "Location")) {
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      if (ctx->location_field_set) {
+        headers_out->location = header;
+      }
+    } else if (STR_EQ_LITERAL(name, "Server")) {
+      headers_out->server = header;
+    } else if (STR_EQ_LITERAL(name, "Content-Length")) {
+      int64 len;
+      CHECK(pagespeed_headers.FindContentLength(&len));
+      headers_out->content_length_n = len;
+      headers_out->content_length = header;
+    } else if (STR_EQ_LITERAL(name, "Content-Encoding")) {
+      headers_out->content_encoding = header;
+    } else if (STR_EQ_LITERAL(name, "Refresh")) {
+      headers_out->refresh = header;
+    } else if (STR_EQ_LITERAL(name, "Content-Range")) {
+      headers_out->content_range = header;
+    } else if (STR_EQ_LITERAL(name, "Accept-Ranges")) {
+      headers_out->accept_ranges = header;
+    } else if (STR_EQ_LITERAL(name, "WWW-Authenticate")) {
+      headers_out->www_authenticate = header;
+    }
+  }
+
+  return NGX_OK;
+}
+
+namespace {
+
+typedef struct {
+  NgxRewriteDriverFactory* driver_factory;
+  MessageHandler* handler;
+} ps_main_conf_t;
+
+typedef struct {
+  // If pagespeed is configured in some server block but not this one our
+  // per-request code will be invoked but server context will be null.  In those
+  // cases we need to short circuit, not changing anything.  Currently our
+  // header filter, body filter, and content handler all do this, but if anyone
+  // adds another way for nginx to give us a request to process we need to check
+  // there as well.
+  NgxServerContext* server_context;
+  ProxyFetchFactory* proxy_fetch_factory;
+  // Only used while parsing config.  After we merge cfg_s and cfg_m you most
+  // likely want cfg_s->server_context->config() as options here will be NULL.
+  NgxRewriteOptions* options;
+  MessageHandler* handler;
+} ps_srv_conf_t;
+
+typedef struct {
+  NgxRewriteOptions* options;
+  MessageHandler* handler;
+} ps_loc_conf_t;
+
+namespace RequestRouting {
+enum Response {
+  kError,
+  kStaticContent,
+  kInvalidUrl,
+  kPagespeedDisabled,
+  kBeacon,
+  kStatistics,
+  kGlobalStatistics,
+  kConsole,
+  kMessages,
+  kAdmin,
+  kCachePurge,
+  kGlobalAdmin,
+  kPagespeedSubrequest,
+  kErrorResponse,
+  kResource,
+};
+}  // namespace RequestRouting
+
+char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
+
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
+// TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
+// not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
+ngx_command_t ps_commands[] = {
+  { ngx_string("pagespeed"),
+    NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
+    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
+    ps_main_configure,
+    NGX_HTTP_SRV_CONF_OFFSET,
+    0,
+    NULL },
+  { ngx_string("pagespeed"),
+    NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
+    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
+    ps_srv_configure,
+    NGX_HTTP_SRV_CONF_OFFSET,
+    0,
+    NULL },
+
+  { ngx_string("pagespeed"),
+    NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF|NGX_CONF_TAKE1|NGX_CONF_MULTI|
+    NGX_CONF_TAKE2|NGX_CONF_TAKE3|NGX_CONF_TAKE4|NGX_CONF_TAKE5,
+    ps_loc_configure,
+    NGX_HTTP_SRV_CONF_OFFSET,
+    0,
+    NULL },
+
+  ngx_null_command
+};
+
+bool ps_disabled(ps_srv_conf_t* cfg_s) {
+  return cfg_s->server_context == NULL ||
+         cfg_s->server_context->config()->unplugged();
+}
+
+void ps_ignore_sigpipe() {
+  struct sigaction act;
+  ngx_memzero(&act, sizeof(act));
+  act.sa_handler = SIG_IGN;
+  sigemptyset(&act.sa_mask);
+  act.sa_flags = 0;
+  sigaction(SIGPIPE, &act, NULL);
+}
+
+// Given a directory path that pagespeed needs, create it and set permissions so
+// the worker can access, but only if needed.
+char* ps_init_dir(const StringPiece& directive,
+                  const StringPiece& path,
+                  ngx_conf_t* cf) {
+  if (path.size() == 0 || path[0] != '/') {
+    return string_piece_to_pool_string(
+        cf->pool, net_instaweb::StrCat(directive, " ", path,
+                                       " must start with a slash"));
+  }
+
+  net_instaweb::StdioFileSystem file_system;
+  net_instaweb::NullMessageHandler message_handler;
+  GoogleString gs_path;
+  path.CopyToString(&gs_path);
+  if (!file_system.IsDir(gs_path.c_str(), &message_handler).is_true()) {
+    if (!file_system.RecursivelyMakeDir(path, &message_handler)) {
+      return string_piece_to_pool_string(
+          cf->pool, net_instaweb::StrCat(
+              directive, " path ", path,
+              " does not exist and could not be created."));
+    }
+    // Directory created, but may not be readable by the worker processes.
+  }
+
+  if (geteuid() != 0) {
+    return NULL;  // We're not root, so we're staying whoever we are.
+  }
+
+  // chown if owner differs from nginx worker user.
+  ngx_core_conf_t* ccf = reinterpret_cast<ngx_core_conf_t*>(
+      ngx_get_conf(cf->cycle->conf_ctx, ngx_core_module));
+  CHECK(ccf != NULL);
+  struct stat gs_stat;
+  if (stat(gs_path.c_str(), &gs_stat) != 0) {
+    return string_piece_to_pool_string(
+        cf->pool, net_instaweb::StrCat(
+            directive, " ", path, " stat() failed"));
+  }
+  if (gs_stat.st_uid != ccf->user) {
+    if (chown(gs_path.c_str(), ccf->user, ccf->group) != 0) {
+      return string_piece_to_pool_string(
+          cf->pool, net_instaweb::StrCat(
+              directive, " ", path, " unable to set permissions"));
+    }
+  }
+
+  return NULL;
+}
+
+// We support interpretation of nginx variables in some configuration settings,
+// but we also need to support literal dollar signs in those same settings.
+// Nginx has no good solution for this, so we define $dollar to expand to '$',
+// which lets people include literal dollar signs if they need them.
+ngx_int_t ps_dollar(
+    ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) {
+  v->valid = 1;
+  v->no_cacheable = 0;
+  v->not_found = 0;
+  v->data = reinterpret_cast<u_char*>(const_cast<char*>("$"));
+  v->len = 1;
+  return NGX_OK;
+}
+
+// Parse the configuration option represented by cf and add it to options,
+// creating options if necessary.
+char* ps_configure(ngx_conf_t* cf,
+                   NgxRewriteOptions** options,
+                   MessageHandler* handler,
+                   net_instaweb::RewriteOptions::OptionScope option_scope) {
+  // args[0] is always "pagespeed"; ignore it.
+  ngx_uint_t n_args = cf->args->nelts - 1;
+
+  // In ps_commands we only register 'pagespeed' as taking up to
+  // five arguments, so this check should never fire.
+  CHECK(n_args <= NGX_PAGESPEED_MAX_ARGS);
+  StringPiece args[NGX_PAGESPEED_MAX_ARGS];
+
+  ngx_str_t* value = static_cast<ngx_str_t*>(cf->args->elts);
+  ngx_uint_t i;
+  for (i = 0 ; i < n_args ; i++) {
+    args[i] = str_to_string_piece(value[i+1]);
+  }
+
+  if (n_args == 1) {
+    if (StringCaseEqual(args[0], "on")) {
+      // safe to call if the setter is disabled
+      g_gzip_setter.EnableGZipForLocation(cf);
+    } else if (StringCaseEqual(args[0], "off")) {
+      g_gzip_setter.SetGZipForLocation(cf, false);
+    }
+  }
+  if (n_args == 2 && args[0].compare("gzip") == 0) {
+    if (args[1].compare("on") == 0) {
+      g_gzip_setter.SetGZipForLocation(cf, true);
+    } else if (args[1].compare("off") == 0) {
+      g_gzip_setter.SetGZipForLocation(cf, false);
+    } else {
+      char* error_message = string_piece_to_pool_string(
+          cf->pool, StringPiece("Invalid pagespeed gzip setting"));
+      return error_message;
+    }
+    return NGX_CONF_OK;
+  }
+
+  // Some options require the worker process to be able to read and write to
+  // a specific directory.  Generally the master process is root while the
+  // worker is nobody, so we need to change permissions and create the directory
+  // if necessary.
+  if (n_args == 2 &&
+      (net_instaweb::StringCaseEqual("LogDir", args[0]) ||
+       net_instaweb::StringCaseEqual("FileCachePath", args[0]))) {
+    char* error_message = ps_init_dir(args[0], args[1], cf);
+    if (error_message != NULL) {
+      return error_message;
+    }
+    // The directory has been prepared, but we haven't actually parsed the
+    // directive yet.  That happens below in ParseAndSetOptions().
+  }
+
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cf->cycle, ngx_pagespeed));
+  if (*options == NULL) {
+    *options = new NgxRewriteOptions(cfg_m->driver_factory->thread_system());
+  }
+
+  ProcessScriptVariablesMode script_mode =
+      dynamic_cast<NgxRewriteDriverFactory*>(cfg_m->driver_factory)
+          ->process_script_variables();
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // To be able to use '$', we map '$ps_dollar' to '$' via a script variable.
+    ngx_str_t name = ngx_string("ps_dollar");
+    ngx_http_variable_t* var =
+        ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE);
+
+    if (var == NULL) {
+      return const_cast<char*>(
+          "Failed to add global configuration variable for '$ps_dollar'");
+    }
+    var->get_handler = ps_dollar;
+  }
+
+  const char* status = (*options)->ParseAndSetOptions(
+      args, n_args, cf->pool, handler, cfg_m->driver_factory, option_scope, cf,
+      script_mode);
+
+  // nginx expects us to return a string literal but doesn't mark it const.
+  return const_cast<char*>(status);
+}
+
+char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
+  return ps_configure(cf, &cfg_s->options, cfg_s->handler,
+                      net_instaweb::RewriteOptions::kProcessScopeStrict);
+}
+
+char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
+  return ps_configure(cf, &cfg_s->options, cfg_s->handler,
+                      net_instaweb::RewriteOptions::kServerScope);
+}
+
+char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+  ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(
+      ngx_http_conf_get_module_loc_conf(cf, ngx_pagespeed));
+  return ps_configure(cf, &cfg_l->options, cfg_l->handler,
+                      net_instaweb::RewriteOptions::kDirectoryScope);
+}
+
+void ps_cleanup_loc_conf(void* data) {
+  ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(data);
+  delete cfg_l->handler;
+  cfg_l->handler = NULL;
+  delete cfg_l->options;
+  cfg_l->options = NULL;
+}
+
+bool factory_deleted = false;
+void ps_cleanup_srv_conf(void* data) {
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(data);
+
+  // destroy the factory on the first call, causing all worker threads
+  // to be shut down when we destroy any proxy_fetch_factories. This
+  // will prevent any queued callbacks to destroyed proxy fetch factories
+  // from being executed
+  if (!factory_deleted && cfg_s->server_context != NULL) {
+    if (active_driver_factory == cfg_s->server_context->factory()) {
+      active_driver_factory = NULL;
+    }
+    delete cfg_s->server_context->factory();
+    factory_deleted = true;
+  }
+  if (cfg_s->proxy_fetch_factory != NULL) {
+    delete cfg_s->proxy_fetch_factory;
+    cfg_s->proxy_fetch_factory = NULL;
+  }
+  delete cfg_s->handler;
+  cfg_s->handler = NULL;
+  delete cfg_s->options;
+  cfg_s->options = NULL;
+}
+
+void ps_cleanup_main_conf(void* data) {
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(data);
+  delete cfg_m->handler;
+  cfg_m->handler = NULL;
+  NgxRewriteDriverFactory::Terminate();
+  NgxRewriteOptions::Terminate();
+
+  // reset the factory deleted flag, so we will clean up properly next time,
+  // in case of a configuration reload.
+  // TODO(oschaaf): get rid of the factory_deleted flag
+  factory_deleted = false;
+}
+
+template <typename ConfT> ConfT* ps_create_conf(ngx_conf_t* cf) {
+  ConfT* cfg = static_cast<ConfT*>(ngx_pcalloc(cf->pool, sizeof(ConfT)));
+  if (cfg == NULL) {
+    return NULL;
+  }
+  cfg->handler = new GoogleMessageHandler();
+  return cfg;
+}
+
+void ps_set_conf_cleanup_handler(
+    ngx_conf_t* cf, void (func)(void*), void* data) {                // NOLINT
+  ngx_pool_cleanup_t* cleanup_m = ngx_pool_cleanup_add(cf->pool, 0);
+  if (cleanup_m == NULL) {
+    ngx_conf_log_error(
+        NGX_LOG_ERR, cf, 0, "failed to register a cleanup handler");
+  } else {
+    cleanup_m->handler = func;
+    cleanup_m->data = data;
+  }
+}
+
+void terminate_process_context() {
+  if (active_driver_factory != NULL) {
+    // If we got here, that means we are in the cache loader/manager
+    // or did not get a chance to cleanup otherwise.
+    delete active_driver_factory;
+    active_driver_factory = NULL;
+    NgxBaseFetch::Terminate();
+  }
+  delete process_context;
+  process_context = NULL;
+}
+
+void* ps_create_main_conf(ngx_conf_t* cf) {
+  if (!process_context_cleanup_hooked) {
+    SystemRewriteDriverFactory::InitApr();
+    atexit(terminate_process_context);
+    process_context_cleanup_hooked = true;
+  }
+  ps_main_conf_t* cfg_m = ps_create_conf<ps_main_conf_t>(cf);
+  if (cfg_m == NULL) {
+    return NGX_CONF_ERROR;
+  }
+  CHECK(!factory_deleted);
+  NgxRewriteOptions::Initialize();
+  NgxRewriteDriverFactory::Initialize();
+
+  cfg_m->driver_factory = new NgxRewriteDriverFactory(
+      *process_context,
+      new SystemThreadSystem(),
+      "" /* hostname, not used */,
+      -1 /* port, not used */);
+  active_driver_factory = cfg_m->driver_factory;
+  active_driver_factory->LoggingInit(ngx_cycle->log, false);
+  cfg_m->driver_factory->Init();
+  ps_set_conf_cleanup_handler(cf, ps_cleanup_main_conf, cfg_m);
+  return cfg_m;
+}
+
+void* ps_create_srv_conf(ngx_conf_t* cf) {
+  ps_srv_conf_t* cfg_s = ps_create_conf<ps_srv_conf_t>(cf);
+  if (cfg_s == NULL) {
+    return NGX_CONF_ERROR;
+  }
+  ps_set_conf_cleanup_handler(cf, ps_cleanup_srv_conf, cfg_s);
+  return cfg_s;
+}
+
+void* ps_create_loc_conf(ngx_conf_t* cf) {
+  ps_loc_conf_t* cfg_l = ps_create_conf<ps_loc_conf_t>(cf);
+  if (cfg_l == NULL) {
+    return NGX_CONF_ERROR;
+  }
+  ps_set_conf_cleanup_handler(cf, ps_cleanup_loc_conf, cfg_l);
+  return cfg_l;
+}
+
+// nginx has hierarchical configuration.  It maintains configurations at many
+// levels.  At various points it needs to merge configurations from different
+// levels, and then it calls this.  First it creates the configuration at the
+// new level, parsing any pagespeed directives, then it merges in the
+// configuration from the level above.  This function should merge the parent
+// configuration into the child.  It's more complex than options->Merge() both
+// because of the cases where the parent or child didn't have any pagespeed
+// directives and because merging is order-dependent in the opposite way we'd
+// like.
+void ps_merge_options(NgxRewriteOptions* parent_options,
+                      NgxRewriteOptions** child_options) {
+  if (parent_options == NULL) {
+    // Nothing to do.
+  } else if (*child_options == NULL) {
+    *child_options = parent_options->Clone();
+  } else {  // Both non-null.
+    // Unfortunately, merging configuration options is order dependent.  We'd
+    // like to just do (*child_options)->Merge(*parent_options)
+    // but then if we had:
+    //    pagespeed RewriteLevel PassThrough
+    //    server {
+    //       pagespeed RewriteLevel CoreFilters
+    //    }
+    // it would always be stuck on PassThrough.
+    NgxRewriteOptions* child_specific_options = *child_options;
+    *child_options = parent_options->Clone();
+    (*child_options)->Merge(*child_specific_options);
+
+    if (child_specific_options->clear_inherited_scripts()) {
+      // We don't want to inherit any inherited script lines from the parent
+      // options here, so we just stick to the child specific ones.
+      child_specific_options->CopyScriptLinesTo(*child_options);
+    } else {
+      // We append the child specific script lines to the parent's script lines
+      // so we preserve the order in which they will be executed at request time
+      child_specific_options->AppendScriptLinesTo(*child_options);
+    }
+    delete child_specific_options;
+  }
+}
+
+namespace {
+
+int times_ps_merge_srv_conf_called = 0;
+
+}  // namespace
+
+// Called exactly once per server block to merge the main configuration with the
+// configuration for this server.
+char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
+  times_ps_merge_srv_conf_called += 1;
+
+  ps_srv_conf_t* parent_cfg_s = static_cast<ps_srv_conf_t*>(parent);
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(child);
+
+  ps_merge_options(parent_cfg_s->options, &cfg_s->options);
+
+  if (cfg_s->options == NULL) {
+    return NGX_CONF_OK;  // No pagespeed options; don't do anything.
+  }
+
+  // ServerContext needs a hostname and port, but I don't see how to get this
+  // and it ignores that a server can have multiple names and ports.  Because
+  // the server context only needs them to make a unique identifier and to make
+  // debugging easier, substitute our own unique identifier.
+  // TODO(jefftk): either figure out how to get a hostname and port for this
+  // server block or change ServerContext not to ask for them.
+  int dummy_port = -times_ps_merge_srv_conf_called;
+
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
+  cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
+  cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
+      "dummy_hostname", dummy_port);
+
+#if (NGX_HTTP_V2)
+  // Save the variable index of the "http2" variable, so we can use it
+  // at request time to lookup whether that's on. We do this conditionally
+  // since NGINX will complain to the user (at [emerg] level!) if it doesn't
+  // know about it.
+  ngx_str_t name = ngx_string("http2");
+  cfg_s->server_context->set_ngx_http2_variable_index(
+      ngx_http_get_variable_index(cf, &name));
+#endif
+
+  // The server context sets some options when we call global_options(). So
+  // let it do that, then merge in options we got from the config file.
+  // Once we do that we're done with cfg_s->options.
+  cfg_s->server_context->global_options()->Merge(*cfg_s->options);
+  NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(
+      cfg_s->server_context->global_options());
+  cfg_s->options->CopyScriptLinesTo(ngx_options);
+  delete cfg_s->options;
+  cfg_s->options = NULL;
+
+  if (!cfg_s->server_context->global_options()->unplugged()) {
+    // Validate FileCachePath
+    GoogleMessageHandler handler;
+    const char* file_cache_path =
+        cfg_s->server_context->config()->file_cache_path().c_str();
+    if (file_cache_path[0] == '\0') {
+      if (!cfg_s->server_context->global_options()->standby()) {
+        return const_cast<char*>("FileCachePath must be set, even for standby");
+      } else {
+        return const_cast<char*>("FileCachePath must be set");
+      }
+    } else if (!cfg_m->driver_factory->file_system()->IsDir(
+        file_cache_path, &handler).is_true()) {
+      return const_cast<char*>(
+          "FileCachePath must be an nginx-writeable directory");
+    }
+  }
+
+  return NGX_CONF_OK;
+}
+
+char* ps_merge_loc_conf(ngx_conf_t* cf, void* parent, void* child) {
+  ps_loc_conf_t* cfg_l = static_cast<ps_loc_conf_t*>(child);
+  if (cfg_l->options == NULL) {
+    // No directory specific options.
+    return NGX_CONF_OK;
+  }
+
+  // While you can't put a "location" block inside a "location" block you can
+  // put an "if" block inside a "location" block, which is implemented by making
+  // a pretend "location" block.  In this case we may have pagespeed options
+  // from the parent "location" block as well as from the current locationish
+  // "if" block.
+  ps_loc_conf_t* parent_cfg_l = static_cast<ps_loc_conf_t*>(parent);
+  if (parent_cfg_l->options != NULL) {
+    // Rebase our options off of the ones defined in the parent location block.
+    ps_merge_options(parent_cfg_l->options, &cfg_l->options);
+    return NGX_CONF_OK;
+  }
+
+  // Pagespeed options are defined in this location block, and it either has no
+  // parent (typical case) or is an if block whose parent location block defines
+  // no pagespeed options.  Base our options off of those in the server block.
+
+  ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+      ngx_http_conf_get_module_srv_conf(cf, ngx_pagespeed));
+
+  if (ps_disabled(cfg_s)) {
+    // Pagespeed options cannot be defined only in location blocks.  There must
+    // be at least a single "pagespeed off" in the main block or a server
+    // block.
+    return NGX_CONF_OK;
+  }
+
+  // If we get here we have parent options ("global options") from cfg_s, child
+  // options ("directory specific options") from cfg_l, and no options from
+  // parent_cfg_l.  Rebase the directory specific options on the global options.
+  ps_merge_options(cfg_s->server_context->config(), &cfg_l->options);
+
+  return NGX_CONF_OK;
+}
+
+// _ef_ is a shorthand for ETag Filter
+ngx_http_output_header_filter_pt ngx_http_ef_next_header_filter;
+
+bool ps_is_https(ngx_http_request_t* r) {
+  // Based on ngx_http_variable_scheme.
+#if (NGX_HTTP_SSL)
+  return r->connection->ssl;
+#endif
+  return false;
+}
+
+int ps_determine_port(ngx_http_request_t* r) {
+  // Return -1 if the port isn't specified, the port number otherwise.
+  //
+  // If a Host header was provided, get the host from that.  Otherwise fall back
+  // to the local port of the incoming connection.
+
+  int port = -1;
+  ngx_table_elt_t* host = r->headers_in.host;
+
+  if (host != NULL) {
+    // Host headers can look like:
+    //
+    //   www.example.com        // normal
+    //   www.example.com:8080   // port specified
+    //   127.0.0.1              // IPv4
+    //   127.0.0.1:8080         // IPv4 with port
+    //   [::1]                  // IPv6
+    //   [::1]:8080             // IPv6 with port
+    //
+    // The IPv6 ones are the annoying ones, but the square brackets allow us to
+    // disambiguate.  To find the port number, we can say:
+    //
+    //   1) Take the text after the final colon.
+    //   2) If all of those characters are digits, that's your port number
+    //
+    // In the case of a plain IPv6 address with no port number, the text after
+    // the final colon will include a ']', so we'll stop processing.
+
+    StringPiece host_str = str_to_string_piece(host->value);
+    size_t colon_index = host_str.rfind(":");
+    if (colon_index == host_str.npos) {
+      return -1;
+    }
+    // Strip everything up to and including the final colon.
+    host_str.remove_prefix(colon_index + 1);
+
+    bool ok = StringToInt(host_str, &port);
+    if (!ok) {
+      // Might be malformed port, or just IPv6 with no port specified.
+      return -1;
+    }
+
+    return port;
+  }
+
+  // Based on ngx_http_variable_server_port.
+#if (NGX_HAVE_INET6)
+  if (r->connection->local_sockaddr->sa_family == AF_INET6) {
+    port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
+        r->connection->local_sockaddr)->sin6_port);
+  }
+#endif
+  if (port == -1 /* still need port */) {
+    port = ntohs(reinterpret_cast<struct sockaddr_in*>(
+        r->connection->local_sockaddr)->sin_port);
+  }
+
+  return port;
+}
+}  // namespace
+
+StringPiece ps_determine_host(ngx_http_request_t* r) {
+  StringPiece host = str_to_string_piece(r->headers_in.server);
+  if (host.size() == 0) {
+    // If host is unspecified, perhaps because of a pure HTTP 1.0 "GET /path",
+    // fall back to server IP address.  Based on ngx_http_variable_server_addr.
+    ngx_str_t  s;
+    u_char addr[NGX_SOCKADDR_STRLEN];
+    s.len = NGX_SOCKADDR_STRLEN;
+    s.data = addr;
+    ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &s, 0);
+    if (rc != NGX_OK) {
+      s.len = 0;
+    }
+    host = str_to_string_piece(s);
+  }
+  return host;
+}
+
+namespace {
+
+GoogleString ps_determine_url(ngx_http_request_t* r) {
+  int port = ps_determine_port(r);
+  GoogleString port_string;
+  if ((ps_is_https(r) && (port == 443 || port == -1)) ||
+      (!ps_is_https(r) && (port == 80 || port == -1))) {
+    // No port specifier needed for requests on default ports.
+    port_string = "";
+  } else {
+    port_string = StrCat(":", IntegerToString(port));
+  }
+
+  StringPiece host = ps_determine_host(r);
+
+  return StrCat(ps_is_https(r) ? "https://" : "http://",
+                host, port_string, str_to_string_piece(r->unparsed_uri));
+}
+
+// we are still at pagespeed phase
+ngx_int_t ps_decline_request(ngx_http_request_t* r) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+  CHECK(ctx != NULL);
+
+  ctx->driver->Cleanup();
+  ctx->driver = NULL;
+  ctx->location_field_set = false;
+  ctx->psol_vary_accept_only = false;
+
+  // re init ctx
+  ctx->html_rewrite = true;
+  ctx->in_place = false;
+  ps_release_base_fetch(ctx);
+  ps_set_buffered(r, false);
+
+  r->count++;
+  r->phase_handler++;
+
+  //restore read_event_handler to what it was in ps_async_wait_response
+  r->read_event_handler = ngx_http_block_reading;
+  r->write_event_handler = ngx_http_core_run_phases;
+  ngx_http_core_run_phases(r);
+  ngx_http_run_posted_requests(r->connection);
+  return NGX_DONE;
+}
+
+ngx_int_t ps_async_wait_response(ngx_http_request_t* r) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+  CHECK(ctx != NULL);
+
+  r->count++;
+  // While we wait for PSOL to complete an async operation, there is a chance
+  // that the underlying connection gets closed,  or a http/2 RST_STREAM is
+  // received before the async operation completes. In that case we don't want
+  // to continue processing this flow. So we override the requests's read event
+  // handler with one that will make nginx abort request processing and execute
+  // our cleanup handlers instead of resuming request processing.
+  r->read_event_handler = ngx_http_test_reading;
+  r->write_event_handler = ngx_http_request_empty_handler;
+  ps_set_buffered(r, true);
+  // We don't need to add a timer here, as it will be set by nginx.
+  return NGX_DONE;
+}
+
+// Populate cfg_* with configuration information for this request.
+// Thin wrappers around ngx_http_get_module_*_conf and cast.
+ps_srv_conf_t* ps_get_srv_config(ngx_http_request_t* r) {
+  return static_cast<ps_srv_conf_t*>(
+      ngx_http_get_module_srv_conf(r, ngx_pagespeed));
+}
+ps_loc_conf_t* ps_get_loc_config(ngx_http_request_t* r) {
+  return static_cast<ps_loc_conf_t*>(
+      ngx_http_get_module_loc_conf(r, ngx_pagespeed));
+}
+
+RewriteOptions* ps_determine_remote_options(ps_srv_conf_t* cfg_s) {
+  if (!cfg_s || !cfg_s->server_context ||
+      !cfg_s->server_context->global_options()) {
+    return NULL;
+  }
+  if (!cfg_s->server_context->global_options()
+          ->remote_configuration_url()
+          .empty()) {
+    RewriteOptions* remote_options =
+        cfg_s->server_context->global_options()->Clone();
+    // This fetch is blocking for up to remote_configuration_timeout_ms ms.
+    cfg_s->server_context->GetRemoteOptions(remote_options, false);
+    return remote_options;
+  }
+  return NULL;
+}
+
+// Wrapper around GetQueryOptions()
+RewriteOptions* ps_determine_request_options(
+    ngx_http_request_t* r,
+    const RewriteOptions* domain_options, /* may be null */
+    RequestHeaders* request_headers,
+    ResponseHeaders* response_headers,
+    RequestContextPtr request_context,
+    ps_srv_conf_t* cfg_s,
+    GoogleUrl* url,
+    GoogleString* pagespeed_query_params,
+    GoogleString* pagespeed_option_cookies) {
+  // Sets option from request headers and url.
+  RewriteQuery rewrite_query;
+  if (!cfg_s->server_context->GetQueryOptions(
+          request_context, domain_options, url, request_headers,
+          response_headers, &rewrite_query)) {
+    // Failed to parse query params or request headers.  Treat this as if there
+    // were no query params given.
+    ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+                  "ps_determine_request_options: parsing headers or query params failed.");
+    return NULL;
+  }
+
+  *pagespeed_query_params =
+      rewrite_query.pagespeed_query_params().ToEscapedString();
+  *pagespeed_option_cookies =
+      rewrite_query.pagespeed_option_cookies().ToEscapedString();
+
+  // Will be NULL if there aren't any options set with query params or in
+  // headers.
+  return rewrite_query.ReleaseOptions();
+}
+
+// Check whether this visitor is already in an experiment.  If they're not,
+// classify them into one by setting a cookie.  Then set options appropriately
+// for their experiment.
+//
+// See InstawebContext::SetExperimentStateAndCookie()
+bool ps_set_experiment_state_and_cookie(ngx_http_request_t* r,
+                                        RequestHeaders* request_headers,
+                                        RewriteOptions* options,
+                                        const StringPiece& host) {
+  CHECK(options->running_experiment());
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  bool need_cookie =
+      cfg_s->server_context->experiment_matcher()->ClassifyIntoExperiment(
+          *request_headers, *cfg_s->server_context->user_agent_matcher(),
+          options);
+  if (need_cookie && host.length() > 0) {
+    PosixTimer timer;
+    int64 time_now_ms = timer.NowMs();
+    int64 expiration_time_ms = (time_now_ms +
+                                options->experiment_cookie_duration_ms());
+
+    // TODO(jefftk): refactor SetExperimentCookie to expose the value we want to
+    // set on the cookie.
+    int state = options->experiment_id();
+    GoogleString expires;
+    ConvertTimeToString(expiration_time_ms, &expires);
+    GoogleString value = StringPrintf(
+        "%s=%s; Expires=%s; Domain=.%s; Path=/",
+        experiment::kExperimentCookie,
+        experiment::ExperimentStateToCookieString(state).c_str(),
+        expires.c_str(), host.as_string().c_str());
+
+    // Set the PagespeedExperiment cookie.
+    ngx_table_elt_t* cookie = static_cast<ngx_table_elt_t*>(
+        ngx_list_push(&r->headers_out.headers));
+    if (cookie == NULL) {
+      return false;
+    }
+    cookie->hash = 1;  // Include this header in the response.
+
+    ngx_str_set(&cookie->key, "Set-Cookie");
+    // It's not safe to use value.c_str here because cookie header only keeps a
+    // pointer to the string data.
+    cookie->value.data = reinterpret_cast<u_char*>(
+        string_piece_to_pool_string(r->pool, value));
+    cookie->value.len = value.size();
+  }
+  return true;
+}
+
+// There are many sources of options:
+//  - the request (query parameters, headers, and cookies)
+//  - location block
+//  - global server options
+//  - experiment framework
+// Consider them all, returning appropriate options for this request, of which
+// the caller takes ownership.  If the only applicable options are global,
+// set options to NULL so we can use server_context->global_options().
+bool ps_determine_options(ngx_http_request_t* r,
+                          RequestHeaders* request_headers,
+                          ResponseHeaders* response_headers,
+                          RewriteOptions** options,
+                          RequestContextPtr request_context,
+                          ps_srv_conf_t* cfg_s,
+                          GoogleUrl* url,
+                          GoogleString* pagespeed_query_params,
+                          GoogleString* pagespeed_option_cookies,
+                          bool html_rewrite) {
+  ps_loc_conf_t* cfg_l = ps_get_loc_config(r);
+
+  // Global options for this server.  Never null.
+  RewriteOptions* global_options = cfg_s->server_context->global_options();
+
+  // Directory-specific options, usually null.  They've already been rebased off
+  // of the global options as part of the configuration process.
+  RewriteOptions* directory_options = cfg_l->options;
+
+  // Request-specific options, nearly always null.  If set they need to be
+  // rebased on the directory options or the global options.
+  RewriteOptions* request_options = ps_determine_request_options(
+      r, directory_options, request_headers, response_headers, request_context,
+      cfg_s, url, pagespeed_query_params, pagespeed_option_cookies);
+  bool have_request_options = request_options != NULL;
+
+  // Because the caller takes ownership of any options we return, the only
+  // situation in which we can avoid allocating a new RewriteOptions is if the
+  // global options are ok as they are and we don't have script variables we
+  // need to evaluate at this point.
+  NgxRewriteOptions* ngx_global_options =
+      dynamic_cast<NgxRewriteOptions*>(global_options);
+  if (!have_request_options && directory_options == NULL &&
+      !global_options->running_experiment() &&
+      ngx_global_options->script_lines().size() == 0) {
+    return true;
+  }
+
+  // Start with directory options if we have them, otherwise request options.
+  if (directory_options != NULL) {
+    if (*options != NULL) {
+      (*options)->Merge(*directory_options);
+    } else {
+      *options = directory_options->Clone();
+    }
+  } else {
+    if (*options == NULL) {
+      *options = global_options->Clone();
+    }
+  }
+
+  NgxRewriteDriverFactory* ngx_factory =
+      dynamic_cast<NgxRewriteDriverFactory*>(cfg_s->server_context->factory());
+  NgxRewriteOptions* ngx_options = dynamic_cast<NgxRewriteOptions*>(*options);
+
+  // ExecuteScriptVariables() sets 'pagespeed off' on ngx_options when execution
+  // fails and then returns false. When that happens we return, as we don't want
+  // to allow enabling pagespeed by request and execute without the intended
+  // configuration.
+  if (!ngx_options->ExecuteScriptVariables(r, cfg_s->handler, ngx_factory)) {
+    return false;
+  }
+
+  // Modify our options in response to request options if specified.
+  if (have_request_options) {
+    (*options)->Merge(*request_options);
+    delete request_options;
+    request_options = NULL;
+  }
+
+  // If we're running an experiment and processing html then modify our options
+  // in response to the experiment.  Except we generally don't want experiments
+  // to be contaminated with unexpected settings, so ignore experiments if we
+  // have request-specific options.  Unless EnrollExperiment is on, probably set
+  // by a query parameter, in which case we want to go ahead and apply the
+  // experimental settings even if it means bad data, because we're just seeing
+  // what it looks like.
+  if ((*options)->running_experiment() &&
+      html_rewrite &&
+      (!have_request_options ||
+       (*options)->enroll_experiment())) {
+    bool ok = ps_set_experiment_state_and_cookie(
+        r, request_headers, *options, url->Host());
+    if (!ok) {
+      delete *options;
+      *options = NULL;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+// Fix URL based on X-Forwarded-Proto.
+// http://code.google.com/p/modpagespeed/issues/detail?id=546 For example, if
+// Apache gives us the URL "http://www.example.com/" and there is a header:
+// "X-Forwarded-Proto: https", then we update this base URL to
+// "https://www.example.com/".  This only ever changes the protocol of the url.
+//
+// Returns true if it modified url, false otherwise.
+bool ps_apply_x_forwarded_proto(ngx_http_request_t* r, GoogleString* url) {
+  // First check for an X-Forwarded-Proto header.
+  const ngx_str_t* x_forwarded_proto_header = NULL;
+
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_in.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (STR_CASE_EQ_LITERAL(header->key, "X-Forwarded-Proto")) {
+      x_forwarded_proto_header = &header->value;
+      break;
+    }
+  }
+
+  if (x_forwarded_proto_header == NULL) {
+    return false;  // No X-Forwarded-Proto header found.
+  }
+
+  StringPiece x_forwarded_proto =
+      str_to_string_piece(*x_forwarded_proto_header);
+  if (!STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "http") &&
+      !STR_CASE_EQ_LITERAL(*x_forwarded_proto_header, "https")) {
+    LOG(WARNING) << "Unsupported X-Forwarded-Proto: " << x_forwarded_proto
+                 << " for URL " << url << " protocol not changed.";
+    return false;
+  }
+
+  StringPiece url_sp(*url);
+  StringPiece::size_type colon_pos = url_sp.find(":");
+
+  if (colon_pos == StringPiece::npos) {
+    return false;  // URL appears to have no protocol; give up.
+  }
+
+  // Replace URL protocol with that specified in X-Forwarded-Proto.
+  *url = StrCat(x_forwarded_proto, url_sp.substr(colon_pos));
+
+  return true;
+}
+
+bool is_pagespeed_subrequest(ngx_http_request_t* r) {
+  ngx_table_elt_t* user_agent_header = r->headers_in.user_agent;
+  if (user_agent_header == NULL) {
+    return false;
+  }
+  StringPiece user_agent = str_to_string_piece(user_agent_header->value);
+  return (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos);
+}
+
+void ps_release_base_fetch(ps_request_ctx_t* ctx) {
+  // In the normal flow BaseFetch doesn't delete itself in HandleDone() because
+  // we still need to receive notification via pipe and call
+  // CollectAccumulatedWrites.  If there's an error and we're cleaning up early
+  // then HandleDone() hasn't been called yet and we need the base fetch to wait
+  // for that and then delete itself.
+  if (ctx->base_fetch != NULL) {
+    ctx->base_fetch->Detach();
+    ctx->base_fetch = NULL;
+  }
+}
+
+// TODO(chaizhenhua): merge into NgxBaseFetch ctor
+void ps_create_base_fetch(StringPiece url,
+                          ps_request_ctx_t* ctx,
+                          RequestContextPtr request_context,
+                          RequestHeaders* request_headers,
+                          NgxBaseFetchType type,
+                          const RewriteOptions* options) {
+  CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
+
+  ngx_http_request_t* r = ctx->r;
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+
+  // Handles its own deletion.  We need to call Release when we're done with
+  // it, and call Done() on the associated parent (Proxy or Resource) fetch. If
+  // we fail before creating the associated fetch then we need to call Done() on
+  // the BaseFetch ourselves.
+  ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
+                                     ctx->preserve_caching_headers, type,
+                                     options);
+  ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
+}
+
+void ps_release_request_context(void* data) {
+  ps_request_ctx_t* ctx = static_cast<ps_request_ctx_t*>(data);
+
+  // proxy_fetch deleted itself if we called Done(), but if an error happened
+  // before then we need to tell it to delete itself.
+  //
+  // If this is a resource fetch then proxy_fetch was never initialized.
+  if (ctx->proxy_fetch != NULL) {
+    ctx->proxy_fetch->Done(false /* failure */);
+    ctx->proxy_fetch = NULL;
+  }
+
+  if (ctx->inflater_ != NULL) {
+    delete ctx->inflater_;
+    ctx->inflater_ = NULL;
+  }
+
+  if (ctx->driver != NULL) {
+    ctx->driver->Cleanup();
+    ctx->driver = NULL;
+  }
+
+  if (ctx->recorder != NULL) {
+    // Deletes recorder.
+    ctx->recorder->DoneAndSetHeaders(NULL, false /* incomplete response */);
+    ctx->recorder = NULL;
+  }
+
+  ps_release_base_fetch(ctx);
+  delete ctx;
+}
+
+// Set us up for processing a request.  Creates a request context and determines
+// which handler should deal with the request.
+RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+
+  if (ps_disabled(cfg_s)) {
+    // Not enabled for this server block.
+    return RequestRouting::kPagespeedDisabled;
+  }
+
+  if (ngx_terminate || ngx_exiting) {
+    return RequestRouting::kError;
+  }
+  if (r->err_status != 0) {
+    return RequestRouting::kErrorResponse;
+  }
+
+  GoogleString url_string = ps_determine_url(r);
+  GoogleUrl url(url_string);
+
+  if (!url.IsWebValid()) {
+    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
+
+    // Let nginx deal with the error however it wants; we will see a NULL ctx in
+    // the body filter or content handler and do nothing.
+    return RequestRouting::kInvalidUrl;
+  }
+
+  if (is_pagespeed_subrequest(r)) {
+    return RequestRouting::kPagespeedSubrequest;
+  } else if (
+      url.PathSansLeaf() == dynamic_cast<NgxRewriteDriverFactory*>(
+          cfg_s->server_context->factory())->static_asset_prefix()) {
+    return RequestRouting::kStaticContent;
+  }
+
+  const NgxRewriteOptions* global_options = cfg_s->server_context->config();
+
+  StringPiece path = url.PathSansQuery();
+  if (StringCaseEqual(path, global_options->statistics_path()) &&
+      global_options->StatisticsAccessAllowed(url)) {
+    return RequestRouting::kStatistics;
+  } else if (StringCaseEqual(path, global_options->global_statistics_path()) &&
+             global_options->GlobalStatisticsAccessAllowed(url)) {
+    return RequestRouting::kGlobalStatistics;
+  } else if (StringCaseEqual(path, global_options->console_path()) &&
+             global_options->ConsoleAccessAllowed(url)) {
+    return RequestRouting::kConsole;
+  } else if (StringCaseEqual(path, global_options->messages_path()) &&
+             global_options->MessagesAccessAllowed(url)) {
+    return RequestRouting::kMessages;
+  } else if (
+      // The admin handlers get everything under a path (/path/*) while all the
+      // other handlers only get exact matches (/path).  So match all paths
+      // starting with the handler path.
+      !global_options->admin_path().empty() &&
+      StringCaseStartsWith(path, global_options->admin_path()) &&
+      global_options->AdminAccessAllowed(url)) {
+    return RequestRouting::kAdmin;
+  } else if (!global_options->global_admin_path().empty() &&
+             StringCaseStartsWith(path, global_options->global_admin_path()) &&
+             global_options->GlobalAdminAccessAllowed(url)) {
+    return RequestRouting::kGlobalAdmin;
+  } else if (global_options->enable_cache_purge() &&
+             !global_options->purge_method().empty() &&
+             (global_options->purge_method() ==
+              str_to_string_piece(r->method_name))) {
+    return RequestRouting::kCachePurge;
+  }
+
+  const GoogleString* beacon_url;
+  if (ps_is_https(r)) {
+    beacon_url = &(global_options->beacon_url().https);
+  } else {
+    beacon_url = &(global_options->beacon_url().http);
+  }
+
+  if (url.PathSansQuery() == StringPiece(*beacon_url)) {
+    return RequestRouting::kBeacon;
+  }
+
+  return RequestRouting::kResource;
+}
+
+ngx_int_t ps_resource_handler(ngx_http_request_t* r,
+                              bool html_rewrite,
+                              RequestRouting::Response response_category) {
+  if (r != r->main) {
+    return NGX_DECLINED;
+  }
+
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  if (ngx_terminate || ngx_exiting) {
+    cfg_s->server_context->message_handler()->Message(
+        kInfo, "ps_resource_handler declining: nginx worker is shutting down");
+
+    if (ctx == NULL) {
+      return NGX_DECLINED;
+    }
+    ps_release_base_fetch(ctx);
+    return NGX_DECLINED;
+  }
+
+  CHECK(!(html_rewrite && (ctx == NULL || ctx->html_rewrite == false)));
+
+  if (!html_rewrite &&
+      r->method != NGX_HTTP_GET &&
+      r->method != NGX_HTTP_HEAD &&
+      r->method != NGX_HTTP_POST &&
+      response_category != RequestRouting::kCachePurge) {
+    return NGX_DECLINED;
+  }
+
+  GoogleString url_string = ps_determine_url(r);
+  GoogleUrl url(url_string);
+
+  if (!url.IsWebValid()) {
+    ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "invalid url");
+    return NGX_DECLINED;
+  }
+
+  scoped_ptr<RequestHeaders> request_headers(new RequestHeaders);
+  scoped_ptr<ResponseHeaders> response_headers(new ResponseHeaders);
+
+  copy_request_headers_from_ngx(r, request_headers.get());
+  copy_response_headers_from_ngx(r, response_headers.get());
+
+  RequestContextPtr request_context(
+      cfg_s->server_context->NewRequestContext(r));
+  GoogleString pagespeed_query_params;
+  GoogleString pagespeed_option_cookies;
+  RewriteOptions* options = ps_determine_remote_options(cfg_s);
+  if (!ps_determine_options(r, request_headers.get(), response_headers.get(),
+                            &options, request_context, cfg_s, &url,
+                            &pagespeed_query_params, &pagespeed_option_cookies,
+                            html_rewrite)) {
+    return NGX_ERROR;
+  }
+
+  // Take ownership of custom_options.
+  scoped_ptr<RewriteOptions> custom_options(options);
+  if (options == NULL) {
+    options = cfg_s->server_context->global_options();
+  }
+
+  request_context->set_options(options->ComputeHttpOptions());
+
+  // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
+  // parameters.  Keep url_string in sync with url.
+  url.Spec().CopyToString(&url_string);
+
+  if (cfg_s->server_context->global_options()->respect_x_forwarded_proto()) {
+    bool modified_url = ps_apply_x_forwarded_proto(r, &url_string);
+    if (modified_url) {
+      url.Reset(url_string);
+      CHECK(url.IsWebValid()) << "The output of ps_apply_x_forwarded_proto"
+                              << " should always be a valid url because it only"
+                              << " changes the scheme between http and https.";
+    }
+  }
+
+  bool pagespeed_resource =
+      !html_rewrite && cfg_s->server_context->IsPagespeedResource(url);
+  bool is_an_admin_handler =
+      response_category == RequestRouting::kStatistics ||
+      response_category == RequestRouting::kGlobalStatistics ||
+      response_category == RequestRouting::kConsole ||
+      response_category == RequestRouting::kAdmin ||
+      response_category == RequestRouting::kGlobalAdmin ||
+      response_category == RequestRouting::kCachePurge;
+
+  // Normally if we're disabled we won't handle any requests, but if we're in
+  // standby mode we do want to handle requests for .pagespeed. resources.
+  if (options->unplugged() ||
+      (!options->enabled() && !pagespeed_resource)) {
+    // Disabled via query params or request headers.
+    return NGX_DECLINED;
+  }
+
+  if (!html_rewrite) {
+    // create request ctx
+    CHECK(ctx == NULL);
+    ctx = new ps_request_ctx_t();
+
+    ctx->r = r;
+    ctx->html_rewrite = false;
+    ctx->in_place = false;
+    ctx->follow_flushes = options->follow_flushes();
+    ctx->preserve_caching_headers = kDontPreserveHeaders;
+
+    // See build_context_for_request() in mod_instaweb.cc
+    // TODO(jefftk): Is this the right place to be modifying caching headers for
+    // html fetches?  Or should that be done later, in the headers flow for
+    // filter mode, rather than here in resource fetch mode?
+    if (!options->modify_caching_headers()) {
+      ctx->preserve_caching_headers = kPreserveAllCachingHeaders;
+    } else if (!options->IsDownstreamCacheIntegrationEnabled()) {
+      // Downstream cache integration is not enabled. Disable original
+      // Cache-Control headers.
+      ctx->preserve_caching_headers = kDontPreserveHeaders;
+    } else if (!pagespeed_resource && !is_an_admin_handler) {
+      ctx->preserve_caching_headers = kPreserveOnlyCacheControl;
+      // Downstream cache integration is enabled. If a rebeaconing key has been
+      // configured and there is a ShouldBeacon header with the correct key,
+      // disable original Cache-Control headers so that the instrumented page is
+      // served out with no-cache.
+      StringPiece should_beacon(request_headers->Lookup1(kPsaShouldBeacon));
+      if (options->MatchesDownstreamCacheRebeaconingKey(should_beacon)) {
+        ctx->preserve_caching_headers = kDontPreserveHeaders;
+      }
+    }
+
+    ctx->recorder = NULL;
+    ctx->url_string = url_string;
+    ctx->location_field_set = false;
+    ctx->psol_vary_accept_only = false;
+
+    // Set up a cleanup handler on the request.
+    ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
+    if (cleanup == NULL) {
+      ps_release_request_context(ctx);
+      return NGX_ERROR;
+    }
+    cleanup->handler = ps_release_request_context;
+    cleanup->data = ctx;
+    ngx_http_set_ctx(r, ctx, ngx_pagespeed);
+  }
+
+  if (pagespeed_resource) {
+    // TODO(jefftk): Set using_spdy appropriately.  See
+    // ProxyInterface::ProxyRequestCallback
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kPageSpeedResource,
+                         options);
+    ResourceFetch::Start(
+        url,
+        custom_options.release() /* null if there aren't custom options */,
+        cfg_s->server_context, ctx->base_fetch);
+    return ps_async_wait_response(r);
+  } else if (is_an_admin_handler) {
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kAdminPage, options);
+    QueryParams query_params;
+    query_params.ParseFromUrl(url);
+
+    PosixTimer timer;
+    int64 now_ms = timer.NowMs();
+    ctx->base_fetch->response_headers()->SetDateAndCaching(
+        now_ms, 0 /* max-age */, ", no-cache");
+
+    if (response_category == RequestRouting::kStatistics ||
+        response_category == RequestRouting::kGlobalStatistics) {
+      cfg_s->server_context->StatisticsPage(
+          response_category == RequestRouting::kGlobalStatistics,
+          query_params,
+          cfg_s->server_context->config(),
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kConsole) {
+      cfg_s->server_context->ConsoleHandler(
+          *cfg_s->server_context->config(),
+          AdminSite::kStatistics,
+          query_params,
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kAdmin ||
+               response_category == RequestRouting::kGlobalAdmin) {
+      cfg_s->server_context->AdminPage(
+          response_category == RequestRouting::kGlobalAdmin,
+          url,
+          query_params,
+          custom_options == NULL ? cfg_s->server_context->config()
+                                 : custom_options.get(),
+          ctx->base_fetch);
+    } else if (response_category == RequestRouting::kCachePurge) {
+      AdminSite* admin_site = cfg_s->server_context->admin_site();
+      admin_site->PurgeHandler(url_string,
+                               cfg_s->server_context->cache_path(),
+                               ctx->base_fetch);
+    } else {
+      CHECK(false);
+    }
+
+    return ps_async_wait_response(r);
+  } else if (!html_rewrite && response_category == RequestRouting::kResource) {
+    bool is_proxy = false;
+    GoogleString mapped_url;
+    GoogleString host_header;
+
+    if (options->domain_lawyer()->MapOriginUrl(
+            url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
+      ps_create_base_fetch(url.Spec(), ctx, request_context,
+                           request_headers.release(), kPageSpeedProxy, options);
+
+      RewriteDriver* driver;
+      if (custom_options.get() == NULL) {
+        driver = cfg_s->server_context->NewRewriteDriver(
+            ctx->base_fetch->request_context());
+      } else {
+        driver = cfg_s->server_context->NewCustomRewriteDriver(
+            custom_options.release(), ctx->base_fetch->request_context());
+      }
+
+      driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
+      driver->set_pagespeed_query_params(pagespeed_query_params);
+      driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
+      cfg_s->proxy_fetch_factory->StartNewProxyFetch(
+          mapped_url, ctx->base_fetch, driver, NULL /*property_callback*/,
+          NULL /*original_content_fetch*/);
+
+      return ps_async_wait_response(r);
+    }
+
+  }
+
+  if (html_rewrite && options->IsAllowed(url.Spec())) {
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kHtmlTransform, options);
+    // Do not store driver in request_context, it's not safe.
+    RewriteDriver* driver;
+
+    // If we don't have custom options we can use NewRewriteDriver which reuses
+    // rewrite drivers and so is faster because there's no wait to construct
+    // them.  Otherwise we have to build a new one every time.
+
+    if (custom_options.get() == NULL) {
+      driver = cfg_s->server_context->NewRewriteDriver(
+          ctx->base_fetch->request_context());
+    } else {
+      // NewCustomRewriteDriver takes ownership of custom_options.
+      driver = cfg_s->server_context->NewCustomRewriteDriver(
+          custom_options.release(), ctx->base_fetch->request_context());
+    }
+
+    driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
+    driver->set_pagespeed_query_params(pagespeed_query_params);
+    driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
+
+    // TODO(jefftk): FlushEarlyFlow would go here.
+    ProxyFetchPropertyCallbackCollector* property_callback =
+        ProxyFetchFactory::InitiatePropertyCacheLookup(
+            !html_rewrite /* is_resource_fetch */,
+            url,
+            cfg_s->server_context,
+            options,
+            ctx->base_fetch);
+
+    // Will call StartParse etc.  The rewrite driver will take care of deleting
+    // itself if necessary.
+    ctx->proxy_fetch = cfg_s->proxy_fetch_factory->CreateNewProxyFetch(
+        url_string, ctx->base_fetch, driver,
+        property_callback,
+        NULL /* original_content_fetch */);
+    ctx->proxy_fetch->set_trusted_input(true);
+    return NGX_OK;
+  }
+
+  if (options->in_place_rewriting_enabled() &&
+      options->enabled() &&
+      options->IsAllowed(url.Spec())) {
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kIproLookup, options);
+
+    // Do not store driver in request_context, it's not safe.
+    RewriteDriver* driver;
+    if (custom_options.get() == NULL) {
+      driver = cfg_s->server_context->NewRewriteDriver(
+          ctx->base_fetch->request_context());
+    } else {
+      // NewCustomRewriteDriver takes ownership of custom_options.
+      driver = cfg_s->server_context->NewCustomRewriteDriver(
+          custom_options.release(), ctx->base_fetch->request_context());
+    }
+
+    driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
+    ctx->driver = driver;
+
+    cfg_s->server_context->message_handler()->Message(
+        kInfo, "Trying to serve rewritten resource in-place: %s",
+        url_string.c_str());
+
+    ctx->in_place = true;
+    ctx->driver->FetchInPlaceResource(
+        url, false /* proxy_mode */, ctx->base_fetch);
+
+    return ps_async_wait_response(r);
+  }
+
+  // NOTE: We are using the below debug message as is for some of our system
+  // tests. So, be careful about test breakages caused by changing or
+  // removing this line.
+  ngx_log_error(NGX_LOG_DEBUG, r->connection->log, 0,
+                "Passing on content handling for non-pagespeed resource '%s'",
+                url_string.c_str());
+  CHECK(ctx->base_fetch == NULL);
+  // set html_rewrite flag.
+  ctx->html_rewrite = true;
+  return NGX_DECLINED;
+}
+
+// Send each buffer in the chain to the proxy_fetch for optimization.
+// Eventually it will make it's way, optimized, to base_fetch.
+void ps_send_to_pagespeed(ngx_http_request_t* r,
+                          ps_request_ctx_t* ctx,
+                          ps_srv_conf_t* cfg_s,
+                          ngx_chain_t* in) {
+  ngx_chain_t* cur;
+  int last_buf = 0;
+  for (cur = in; cur != NULL; cur = cur->next) {
+    last_buf = cur->buf->last_buf;
+    // Buffers are not really the last buffer until they've been through
+    // pagespeed.
+    cur->buf->last_buf = 0;
+
+    CHECK(ctx->proxy_fetch != NULL);
+    if (ctx->inflater_ == NULL) {
+      ctx->proxy_fetch->Write(
+          StringPiece(reinterpret_cast<char*>(cur->buf->pos),
+                      cur->buf->last - cur->buf->pos), cfg_s->handler);
+    } else {
+      char buf[kStackBufferSize];
+      ctx->inflater_->SetInput(reinterpret_cast<char*>(cur->buf->pos),
+                               cur->buf->last - cur->buf->pos);
+      while (ctx->inflater_->HasUnconsumedInput()) {
+        int num_inflated_bytes = ctx->inflater_->InflateBytes(
+            buf, kStackBufferSize);
+        if (num_inflated_bytes < 0) {
+          cfg_s->handler->Message(kWarning, "Corrupted inflation");
+        } else if (num_inflated_bytes > 0) {
+          ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
+                                  cfg_s->handler);
+        }
+      }
+    }
+    if (cur->buf->flush && ctx->follow_flushes) {
+      // Calling ctx->proxy_fetch->Flush(cfg_s->handler) will be a no-op here,
+      // unless we have follow_flushes or flush_html enabled. Note that PSOL
+      // might aggregate multiple flushes into 1, and actually flush a little bit
+      // later due to html parser state and earlier scheduled operations.
+      // Also, unless we also set the flush flag on the nginx buffers we won't
+      // actually flush.
+      // Note that too many flushes could harm optimization over larger html
+      // fragments as PSOL gets less context to work with, e.g. it can't combine
+      // two css files if a flush happens in between.
+      ctx->proxy_fetch->Flush(cfg_s->handler);
+    }
+
+    // We're done with buffers as we pass them through, so mark them as sent as
+    // we go.
+    cur->buf->pos = cur->buf->last;
+  }
+
+  if (last_buf) {
+    ctx->proxy_fetch->Done(true /* success */);
+    ctx->proxy_fetch = NULL;  // ProxyFetch deletes itself on Done().
+  }
+}
+
+#ifndef ngx_http_clear_etag
+// The ngx_http_clear_etag(r) macro was added in 1.3.3.  Backport it if it's not
+// present.
+#define ngx_http_clear_etag(r)                  \
+  if (r->headers_out.etag) {                    \
+    r->headers_out.etag->hash = 0;              \
+    r->headers_out.etag = NULL;                 \
+  }
+#endif
+
+void ps_strip_html_headers(ngx_http_request_t* r) {
+  // We're modifying content, so switch to 'Transfer-Encoding: chunked' and
+  // calculate on the fly.
+  ngx_http_clear_content_length(r);
+
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    // We also need to strip:
+    //   Accept-Ranges
+    //    - won't work because our html changes
+    //   Vary: Accept-Encoding
+    //    - our gzip filter will add this later
+    if (STR_CASE_EQ_LITERAL(header->key, "Accept-Ranges") ||
+        (STR_CASE_EQ_LITERAL(header->key, "Vary") &&
+         STR_CASE_EQ_LITERAL(header->value, "Accept-Encoding"))) {
+      // Response headers with hash of 0 are excluded from the response.
+      header->hash = 0;
+    }
+  }
+}
+
+// Returns true, if the the response headers indicate there are multiple
+// content encodings.
+bool ps_has_stacked_content_encoding(ngx_http_request_t* r) {
+  ngx_uint_t i;
+  ngx_list_part_t* part = &(r->headers_out.headers.part);
+  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
+  int field_count = 0;
+
+  for (i = 0 ; /* void */; i++) {
+    if (i >= part->nelts) {
+      if (part->next == NULL) {
+        break;
+      }
+
+      part = part->next;
+      header = static_cast<ngx_table_elt_t*>(part->elts);
+      i = 0;
+    }
+
+    // Inspect Content-Encoding headers, checking all value fields
+    // If an origin returns gzip,foo, that is what we will get here.
+    if (STR_CASE_EQ_LITERAL(header[i].key, "Content-Encoding")) {
+      if (header[i].value.data != NULL && header[i].value.len > 0) {
+        char* p = reinterpret_cast<char*>(header[i].value.data);
+        ngx_uint_t j;
+        for (j = 0; j < header[i].value.len; j++) {
+          if (p[j] == ',' || j == header[i].value.len - 1) {
+            field_count++;
+          }
+        }
+        if (field_count > 1) {
+          return true;
+        }
+      }
+    }
+  }
+
+  return false;
+}
+
+ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
+  u_char* etag = reinterpret_cast<u_char*>(
+      const_cast<char*>(kInternalEtagName));
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(r->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->key.len == strlen(kInternalEtagName) &&
+        !ngx_strncasecmp(header->key.data, etag, header->key.len)) {
+      header->key.data = reinterpret_cast<u_char*>(const_cast<char*>("ETag"));
+      header->key.len = 4;
+      r->headers_out.etag = header;
+      break;
+    }
+  }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+#if (NGX_HTTP_GZIP)
+  if (ctx && ctx->psol_vary_accept_only) {
+    r->gzip_vary = 0;
+  }
+#endif
+
+  if (ctx && ctx->recorder) {
+    ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+    int s_maxage_sec =
+        cfg_s->server_context->global_options()->EffectiveInPlaceSMaxAgeSec();
+    if (s_maxage_sec != -1) {
+      GoogleString existing_cache_control;
+      bool cache_control_present = ps_get_cache_control(
+          r, &existing_cache_control);
+      GoogleString updated_cache_control;
+      if (ResponseHeaders::ApplySMaxAge(s_maxage_sec,
+                                        existing_cache_control,
+                                        &updated_cache_control)) {
+        // We're modifing the cache control header; save a copy first.
+        // NULL indicates that the header was not present.
+        ctx->recorder->SaveCacheControl(
+            cache_control_present ? existing_cache_control.c_str() : NULL);
+
+        // Replace the cache-control with our new s-maxage-including one.
+        ps_set_cache_control(r, string_piece_to_pool_string(
+            r->pool, updated_cache_control));
+      }
+    }
+  }
+
+  return ngx_http_ef_next_header_filter(r);
+}
+
+namespace html_rewrite {
+ngx_http_output_header_filter_pt ngx_http_next_header_filter;
+ngx_http_output_body_filter_pt ngx_http_next_body_filter;
+
+// After pagespeed has had a chance to run, copy the headers it produced to
+// nginx so it can send them out to the browser.
+ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  if (ps_disabled(cfg_s)) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (r != r->main) {
+    // Don't handle subrequests.
+    return ngx_http_next_header_filter(r);
+  }
+  // Poll for cache flush on every request (polls are rate-limited).
+  cfg_s->server_context->FlushCacheIfNecessary();
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  if (ctx == NULL || ctx->html_rewrite == false) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (r->err_status != 0) {
+    ctx->html_rewrite = false;
+    return ngx_http_next_header_filter(r);
+  }
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "http pagespeed html rewrite header filter \"%V\"", &r->uri);
+
+  // We don't know what this request is, but we only want to send html through
+  // to pagespeed.  Check the content type header and find out.
+  const ContentType* content_type =
+      MimeTypeToContentType(
+          str_to_string_piece(r->headers_out.content_type));
+  if (content_type == NULL || !content_type->IsHtmlLike()) {
+    // Unknown or otherwise non-html content type: skip it.
+    ctx->html_rewrite = false;
+    return ngx_http_next_header_filter(r);
+  }
+
+  ngx_int_t rc = ps_resource_handler(r, true /* html rewrite */,
+                                     RequestRouting::kResource);
+  if (rc != NGX_OK) {
+    ctx->html_rewrite = false;
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (r->headers_out.content_encoding &&
+      r->headers_out.content_encoding->value.len) {
+    // headers_out.content_encoding will be set to the exact last
+    // Content-Encoding response header value that nginx receives. To
+    // check if there were multiple (aka stacked) encodings in the
+    // response headers, we must iterate them all.
+    if (!ps_has_stacked_content_encoding(r)) {
+      StringPiece content_encoding =
+          str_to_string_piece(r->headers_out.content_encoding->value);
+      GzipInflater::InflateType inflate_type = GzipInflater::kGzip;
+      bool is_encoded = false;
+      if (StringCaseEqual(content_encoding, "deflate")) {
+        is_encoded = true;
+        inflate_type = GzipInflater::kDeflate;
+      } else if (StringCaseEqual(content_encoding, "gzip")) {
+        is_encoded = true;
+      }
+
+      if (is_encoded) {
+        r->headers_out.content_encoding->hash = 0;
+        r->headers_out.content_encoding = NULL;
+        ctx->inflater_ = new GzipInflater(inflate_type);
+        ctx->inflater_->Init();
+      }
+    }
+  }
+
+  ps_strip_html_headers(r);
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/819
+  ctx->location_field_set = r->headers_out.location != NULL;
+
+  // TODO(jefftk): is this thread safe?
+  copy_response_headers_from_ngx(r, ctx->base_fetch->response_headers());
+
+  ps_set_buffered(r, true);
+  r->filter_need_in_memory = 1;
+  return NGX_AGAIN;
+}
+
+ngx_int_t ps_html_rewrite_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  if (ps_disabled(cfg_s)) {
+    return ngx_http_next_body_filter(r, in);
+  }
+
+  if (r != r->main) {
+    // Don't handle subrequests.
+    return ngx_http_next_body_filter(r, in);
+  }
+  // Don't need to check for a cache flush; already did in
+  // ps_html_rewrite_header_filter.
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  if (ctx == NULL || ctx->html_rewrite == false) {
+    // ctx is null iff we've decided to pass through this request unchanged.
+    return ngx_http_next_body_filter(r, in);
+  }
+
+  // We don't want to handle requests with errors, but we should be dealing with
+  // that in the header filter and not initializing ctx.
+  CHECK(r->err_status == 0);                                         // NOLINT
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "http pagespeed html rewrite body filter \"%V\"", &r->uri);
+
+
+  if (in != NULL) {
+    // Send all input data to the proxy fetch.
+    ps_send_to_pagespeed(r, ctx, cfg_s, in);
+  }
+
+  return ngx_http_next_body_filter(r, NULL);
+}
+
+void ps_html_rewrite_filter_init() {
+  ngx_http_next_header_filter = ngx_http_top_header_filter;
+  ngx_http_top_header_filter = ps_html_rewrite_header_filter;
+
+  ngx_http_next_body_filter = ngx_http_top_body_filter;
+  ngx_http_top_body_filter = ps_html_rewrite_body_filter;
+}
+
+}  // namespace html_rewrite
+
+using html_rewrite::ps_html_rewrite_filter_init;
+
+namespace in_place {
+
+ngx_http_output_header_filter_pt ngx_http_next_header_filter;
+ngx_http_output_body_filter_pt ngx_http_next_body_filter;
+
+// Header filter to support IPRO.
+//
+// The control flow here is tricky, probably excessively so.
+ngx_int_t ps_in_place_check_header_filter(ngx_http_request_t* r) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+
+  if (ctx == NULL) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (ctx->recorder != NULL) {
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "ps in place check header filter recording: %V", &r->uri);
+
+    CHECK(!ctx->in_place);
+
+    // We didn't find this resource in cache originally, so we're recording it
+    // as it passes us by.  At this point the headers from things that run
+    // before us are set but not things that run after us, which means here is
+    // where we need to check whether there's a "Content-Encoding: gzip".  If we
+    // waited to do this in ps_in_place_body_filter we wouldn't be able to tell
+    // the difference between response headers that have "C-E: gz" because we're
+    // proxying for an upstream that gzipped the content and response headers
+    // that have it because the gzip filter (which runs after us) is going to
+    // produce gzipped output.
+    //
+    // The recorder will do this checking, so pass it the headers.
+    ResponseHeaders response_headers;
+    copy_response_headers_from_ngx(r, &response_headers);
+    ctx->recorder->ConsiderResponseHeaders(
+        InPlaceResourceRecorder::kPreliminaryHeaders, &response_headers);
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (!ctx->in_place) {
+    return ngx_http_next_header_filter(r);
+  }
+
+  // If our request was finalized during the IPRO lookup, we decline.
+  if (r->connection->error) {
+    return ps_decline_request(r);
+  }
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "ps in place check header filter initial: %V", &r->uri);
+
+  int status_code = r->headers_out.status;
+  bool status_ok = (status_code != 0) && (status_code < 400);
+
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  NgxServerContext* server_context = cfg_s->server_context;
+  MessageHandler* message_handler = cfg_s->handler;
+  GoogleString url = ps_determine_url(r);
+  // The URL we use for cache key is a bit different since it may
+  // have PageSpeed query params removed.
+  GoogleString cache_url = ctx->url_string;
+
+  // continue process
+  if (status_ok) {
+    ctx->in_place = false;
+
+    server_context->rewrite_stats()->ipro_served()->Add(1);
+    message_handler->Message(
+        kInfo, "Serving rewritten resource in-place: %s",
+        url.c_str());
+
+    return ngx_http_next_header_filter(r);
+  }
+
+  if (status_code == CacheUrlAsyncFetcher::kNotInCacheStatus &&
+      !r->header_only) {
+    server_context->rewrite_stats()->ipro_not_in_cache()->Add(1);
+    server_context->message_handler()->Message(
+        kInfo,
+        "Could not rewrite resource in-place "
+        "because URL is not in cache: %s",
+        cache_url.c_str());
+    const SystemRewriteOptions* options = SystemRewriteOptions::DynamicCast(
+        ctx->driver->options());
+
+    RequestContextPtr request_context(
+        cfg_s->server_context->NewRequestContext(r));
+    request_context->set_options(options->ComputeHttpOptions());
+    RequestHeaders request_headers;
+    copy_request_headers_from_ngx(r, &request_headers);
+    // This URL was not found in cache (neither the input resource nor
+    // a ResourceNotCacheable entry) so we need to get it into cache
+    // (or at least a note that it cannot be cached stored there).
+    // We do that using an Apache output filter.
+    ctx->recorder = new InPlaceResourceRecorder(
+        request_context,
+        cache_url,
+        ctx->driver->CacheFragment(),
+        request_headers.GetProperties(),
+        options->ipro_max_response_bytes(),
+        options->ipro_max_concurrent_recordings(),
+        server_context->http_cache(),
+        server_context->statistics(),
+        message_handler);
+    // set in memory flag for in_place_body_filter
+    r->filter_need_in_memory = 1;
+
+    // We don't have the response headers at all yet because we haven't yet gone
+    // to the backend.
+  } else {
+    server_context->rewrite_stats()->ipro_not_rewritable()->Add(1);
+    message_handler->Message(
+        kInfo, "Could not rewrite resource in-place: %s", url.c_str());
+  }
+
+  return ps_decline_request(r);
+}
+
+// If we've decided that we should record this response for future optimization
+// with IPRO, then log the bytes as they come through
+ngx_int_t ps_in_place_body_filter(ngx_http_request_t* r, ngx_chain_t* in) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+  if (ctx == NULL || ctx->recorder == NULL) {
+    return ngx_http_next_body_filter(r, in);
+  }
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "ps in place body filter: %V", &r->uri);
+
+  InPlaceResourceRecorder* recorder = ctx->recorder;
+  for (ngx_chain_t* cl = in; cl; cl = cl->next) {
+    if (ngx_buf_size(cl->buf)) {
+      CHECK(ngx_buf_in_memory(cl->buf));
+      StringPiece contents(reinterpret_cast<char*>(cl->buf->pos),
+                           ngx_buf_size(cl->buf));
+      recorder->Write(contents, recorder->handler());
+    }
+
+    if (cl->buf->flush) {
+      recorder->Flush(recorder->handler());
+    }
+
+    if (cl->buf->last_buf || recorder->failed()) {
+      ResponseHeaders response_headers;
+      copy_response_headers_from_ngx(r, &response_headers);
+      ctx->recorder->DoneAndSetHeaders(
+          &response_headers,
+          cl->buf->last_buf /* response is complete if last_buf is set */);
+      ctx->recorder = NULL;
+      break;
+    }
+  }
+
+  return ngx_http_next_body_filter(r, in);
+}
+
+void ps_in_place_filter_init() {
+  ngx_http_next_header_filter = ngx_http_top_header_filter;
+  ngx_http_top_header_filter = ps_in_place_check_header_filter;
+
+  ngx_http_next_body_filter = ngx_http_top_body_filter;
+  ngx_http_top_body_filter = ps_in_place_body_filter;
+}
+
+}  // namespace in_place
+
+using in_place::ps_in_place_filter_init;
+
+ngx_int_t send_out_headers_and_body(
+    ngx_http_request_t* r,
+    const ResponseHeaders& response_headers,
+    const GoogleString& output) {
+  ngx_int_t rc = copy_response_headers_to_ngx(
+      r, response_headers, kDontPreserveHeaders);
+
+  if (rc != NGX_OK) {
+    return NGX_ERROR;
+  }
+
+  rc = ngx_http_send_header(r);
+
+  if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
+    return rc;
+  }
+
+  // Send the body.
+  ngx_chain_t* out;
+  rc = string_piece_to_buffer_chain(
+      r->pool, output, &out, true /* send_last_buf */, false);
+  if (rc == NGX_ERROR) {
+    return NGX_ERROR;
+  }
+  CHECK(rc == NGX_OK);
+
+  return ngx_http_output_filter(r, out);
+}
+
+// Handle responses where we have the content we need in memory and can just
+// send it right out.
+ngx_int_t ps_simple_handler(ngx_http_request_t* r,
+                            NgxServerContext* server_context,
+                            RequestRouting::Response response_category) {
+  NgxRewriteDriverFactory* factory =
+      static_cast<NgxRewriteDriverFactory*>(
+          server_context->factory());
+  NgxMessageHandler* message_handler = factory->ngx_message_handler();
+  StringPiece request_uri_path = str_to_string_piece(r->uri);
+
+  GoogleString url_string = ps_determine_url(r);
+  GoogleUrl url(url_string);
+  QueryParams query_params;
+  if (url.IsWebValid()) {
+    query_params.ParseFromUrl(url);
+  }
+
+  GoogleString output;
+  StringWriter writer(&output);
+  HttpStatus::Code status = HttpStatus::kOK;
+  ContentType content_type = kContentTypeHtml;
+  StringPiece cache_control = HttpAttributes::kNoCache;
+  const char* error_message = NULL;
+
+  switch (response_category) {
+    case RequestRouting::kStaticContent: {
+      StringPiece file_contents;
+      if (!server_context->static_asset_manager()->GetAsset(
+              request_uri_path.substr(factory->static_asset_prefix().length()),
+              &file_contents, &content_type, &cache_control)) {
+        return NGX_DECLINED;
+      }
+      file_contents.CopyToString(&output);
+      break;
+    }
+    case RequestRouting::kMessages: {
+      GoogleString log;
+      StringWriter log_writer(&log);
+      if (!message_handler->Dump(&log_writer)) {
+        writer.Write("Writing to ngx_pagespeed_message failed. \n"
+                     "Please check if it's enabled in pagespeed.conf.\n",
+                     message_handler);
+      } else {
+        HtmlKeywords::WritePre(log, "", &writer, message_handler);
+      }
+      break;
+    }
+    default:
+      ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                    "ps_simple_handler: unknown RequestRouting.");
+      return NGX_ERROR;
+  }
+
+  if (error_message != NULL) {
+    status = HttpStatus::kNotFound;
+    content_type = kContentTypeHtml;
+    output = error_message;
+  }
+
+  ResponseHeaders response_headers;
+  response_headers.SetStatusAndReason(status);
+  response_headers.set_major_version(1);
+  response_headers.set_minor_version(1);
+
+  response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
+  // http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
+  // Script and styleSheet elements will reject responses with
+  // incorrect MIME types if the server sends the response header
+  // "X-Content-Type-Options: nosniff". This is a security feature
+  // that helps prevent attacks based on MIME-type confusion.
+  response_headers.Add("X-Content-Type-Options", "nosniff");
+
+  int64 now_ms = factory->timer()->NowMs();
+  response_headers.SetDate(now_ms);
+  response_headers.SetLastModified(now_ms);
+  response_headers.Add(HttpAttributes::kCacheControl, cache_control);
+
+  char* cache_control_s = string_piece_to_pool_string(r->pool, cache_control);
+  if (cache_control_s != NULL) {
+    if (FindIgnoreCase(cache_control, "private") == StringPiece::npos) {
+      response_headers.Add(HttpAttributes::kEtag, "W/\"0\"");
+    }
+  }
+
+  return send_out_headers_and_body(r, response_headers, output);
+}
+
+void ps_beacon_handler_helper(ngx_http_request_t* r,
+                              StringPiece beacon_data) {
+  ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                "ps_beacon_handler_helper: beacon[%d] %*s",
+                beacon_data.size(),  beacon_data.size(),
+                beacon_data.data());
+
+  StringPiece user_agent;
+  if (r->headers_in.user_agent != NULL) {
+    user_agent = str_to_string_piece(r->headers_in.user_agent->value);
+  }
+
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  CHECK(cfg_s != NULL);
+
+  RequestContextPtr request_context(
+      cfg_s->server_context->NewRequestContext(r));
+  // TODO(sligocki): Do we want custom options here? It probably doesn't matter
+  // for beacons.
+  request_context->set_options(
+      cfg_s->server_context->global_options()->ComputeHttpOptions());
+
+  cfg_s->server_context->HandleBeacon(beacon_data,
+                                      user_agent,
+                                      request_context);
+
+  ps_set_cache_control(r, const_cast<char*>("max-age=0, no-cache"));
+
+  // TODO(jefftk): figure out how to insert Content-Length:0 as a response
+  // header so wget doesn't hang.
+}
+
+// Load the request body into out.  ngx_http_read_client_request_body must
+// already have been called.  Return false on failure, true on success.
+bool ps_request_body_to_string_piece(
+    ngx_http_request_t* r, StringPiece* out) {
+  if (r->request_body == NULL || r->request_body->bufs == NULL) {
+    ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                  "ps_request_body_to_string_piece: "
+                  "empty request body.");
+    return false;
+  }
+
+  if (r->request_body->temp_file) {
+    u_char buf[POST_BUF_READ_SIZE];
+    int count = 0;
+    ssize_t ret;
+    GoogleString tmp;
+
+    // Note that we depend on nginx to impose sensible limits on post data.
+    while ((ret = ngx_read_file(&r->request_body->temp_file->file,
+                                buf, POST_BUF_READ_SIZE, count)) > 0) {
+      tmp.append(reinterpret_cast<char*>(buf), ret);
+      count += ret;
+    }
+
+    if (ret < 0) {
+      ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                    "ps_request_body_to_string_piece: "
+                    "error reading post body.");
+      return false;
+    }
+
+    // We have read the complete post body, copy it to a buffer
+    // from the request's pool.
+    u_char* data = reinterpret_cast<u_char*>(ngx_palloc(r->pool, count));
+    memcpy(data, tmp.c_str(), count);
+    *out = StringPiece(reinterpret_cast<char*>(data), count);
+
+    return true;
+  } else if (r->request_body->bufs->next == NULL) {
+    // There's just one buffer, so we can simply return a StringPiece pointing
+    // to this buffer.
+    ngx_buf_t* buffer = r->request_body->bufs->buf;
+    CHECK(!buffer->in_file);
+    int len = buffer->last - buffer->pos;
+    ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                  "ngx_pagespeed beacon: single buffer of %d", len);
+    *out = StringPiece(reinterpret_cast<char*>(buffer->pos), len);
+    return true;
+  } else {
+    // There are multiple buffers, so we need to allocate memory for a string to
+    // hold the whole result.  This should only happen when the POST is sent
+    // with "Transfer-Encoding: Chunked".
+
+    // First determine how much data there is.
+    int len = 0;
+    int buffers = 0;
+
+    ngx_chain_t* chain_link;
+    for (chain_link = r->request_body->bufs;
+         chain_link != NULL;
+         chain_link = chain_link->next) {
+      len += chain_link->buf->last - chain_link->buf->pos;
+      buffers++;
+    }
+
+    ngx_log_debug(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                  "ngx_pagespeed beacon: %d buffers totalling %d", len);
+
+    // Allocate a string to store the combined result.
+    u_char* s = static_cast<u_char*>(ngx_palloc(r->pool, len));
+    if (s == NULL) {
+      ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
+                    "ps_request_body_to_string_piece: "
+                    "failed to allocate memory");
+      return false;
+    }
+
+    // Copy the data into the combined string.
+    u_char* current_position = s;
+    int i;
+    for (chain_link = r->request_body->bufs, i = 0;
+         chain_link != NULL;
+         chain_link = chain_link->next, i++) {
+      ngx_buf_t* buffer = chain_link->buf;
+      CHECK(!buffer->in_file);
+      current_position = ngx_copy(current_position, buffer->pos,
+                                  buffer->last - buffer->pos);
+    }
+    CHECK_EQ(current_position, s + len);
+    *out = StringPiece(reinterpret_cast<char*>(s), len);
+    return true;
+  }
+}
+
+// Parses out query params from the request.
+void ps_query_params_handler(ngx_http_request_t* r, StringPiece* data) {
+  StringPiece unparsed_uri = str_to_string_piece(r->unparsed_uri);
+  stringpiece_ssize_type question_mark_index = unparsed_uri.find("?");
+  if (question_mark_index == StringPiece::npos) {
+    *data = "";
+  } else {
+    *data = unparsed_uri.substr(
+        question_mark_index+1, unparsed_uri.size() - (question_mark_index+1));
+  }
+}
+
+// Called after nginx reads the request body from the client.  For another
+// example processing request buffers, see ngx_http_form_input_module.c
+void ps_beacon_body_handler(ngx_http_request_t* r) {
+  // Even if the beacon is a POST, the originating url should be in the query
+  // params, not the POST body.
+  StringPiece query_param_beacon_data;
+  ps_query_params_handler(r, &query_param_beacon_data);
+
+  StringPiece request_body;
+  bool ok = ps_request_body_to_string_piece(r, &request_body);
+  GoogleString beacon_data = StrCat(
+      query_param_beacon_data, "&", request_body);
+  if (ok) {
+    ps_beacon_handler_helper(r, beacon_data.c_str());
+    ngx_http_finalize_request(r, NGX_HTTP_NO_CONTENT);
+  } else {
+    ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR);
+  }
+}
+
+// We need to get the beacon data to ps_beacon_body_handler so it can pass it
+// along to SystemServerContext::HandleBeacon, but it might have been POSTed.
+// If it's posted we need to make an async call to get the data, otherwise just
+// read it out of the query params.
+ngx_int_t ps_beacon_handler(ngx_http_request_t* r) {
+  if (r->method == NGX_HTTP_POST) {
+    // Use post body. Handler functions are called before the request body has
+    // been read from the client, so we need to ask nginx to read it from the
+    // client and then call us back.  Control flow continues in
+    // ps_beacon_body_handler unless there's an error reading the request body.
+    //
+    // See: http://forum.nginx.org/read.php?2,31312,31312
+    ngx_int_t rc = ngx_http_read_client_request_body(r, ps_beacon_body_handler);
+    if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {
+      return rc;
+    }
+    return NGX_DONE;
+  } else {
+    // Use query params.
+    StringPiece query_param_beacon_data;
+    ps_query_params_handler(r, &query_param_beacon_data);
+    ps_beacon_handler_helper(r, query_param_beacon_data);
+    return NGX_HTTP_NO_CONTENT;
+  }
+}
+
+// Some things pagespeed filters on the way past (html) and other things it
+// actually handles, like requests for resources
+// (example.css.pagespeed.ce.LyfcM6Wulf.css) and static content
+// (/ngx_pagespeed_static/js_defer.q1EBmcgYOC.js).  This is called once we know
+// we need to handle a resource, and it figures out which particular handler can
+// supply it to the user.
+ngx_int_t ps_content_handler(ngx_http_request_t* r) {
+  ps_srv_conf_t* cfg_s = ps_get_srv_config(r);
+  if (ps_disabled(cfg_s)) {
+    // Pagespeed is not on for this server block.
+    return NGX_DECLINED;
+  }
+
+  // Poll for cache flush on every request (polls are rate-limited).
+  cfg_s->server_context->FlushCacheIfNecessary();
+
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "http pagespeed handler \"%V\"", &r->uri);
+
+  RequestRouting::Response response_category =
+      ps_route_request(r);
+  switch (response_category) {
+    case RequestRouting::kError:
+      return NGX_ERROR;
+    case RequestRouting::kPagespeedDisabled:
+    case RequestRouting::kInvalidUrl:
+    case RequestRouting::kPagespeedSubrequest:
+    case RequestRouting::kErrorResponse:
+      return NGX_DECLINED;
+    case RequestRouting::kBeacon:
+      return ps_beacon_handler(r);
+    case RequestRouting::kStaticContent:
+    case RequestRouting::kMessages:
+      return ps_simple_handler(r, cfg_s->server_context, response_category);
+    case RequestRouting::kStatistics:
+    case RequestRouting::kGlobalStatistics:
+    case RequestRouting::kConsole:
+    case RequestRouting::kAdmin:
+    case RequestRouting::kGlobalAdmin:
+    case RequestRouting::kCachePurge:
+    case RequestRouting::kResource:
+      return ps_resource_handler(
+          r, false /* html rewrite */, response_category);
+  }
+
+  CHECK(0);
+  return NGX_ERROR;
+}
+
+ngx_int_t ps_phase_handler(ngx_http_request_t* r,
+                           ngx_http_phase_handler_t* ph) {
+  ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                 "pagespeed phase: %ui", r->phase_handler);
+
+  r->write_event_handler = ngx_http_request_empty_handler;
+
+  ngx_int_t rc = ps_content_handler(r);
+  // Warning: this requires ps_content_handler to always return NGX_DECLINED
+  // directly if it's not going to handle the request. It is not ok for
+  // ps_content_handler to asynchronously determine whether to handle the
+  // request, returning NGX_DONE here.
+  if (rc == NGX_DECLINED) {
+    r->write_event_handler = ngx_http_core_run_phases;
+    r->phase_handler++;
+    return NGX_AGAIN;
+  }
+
+  ngx_http_finalize_request(r, rc);
+  return NGX_OK;
+}
+
+namespace fix_headers {
+ngx_http_output_header_filter_pt ngx_http_next_header_filter;
+
+// Clear a few headers from html responses.
+
+ngx_int_t ps_html_rewrite_fix_headers_filter(ngx_http_request_t* r) {
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+  if (r != r->main || ctx == NULL || !ctx->html_rewrite ||
+      ctx->preserve_caching_headers == kPreserveAllCachingHeaders) {
+    return ngx_http_next_header_filter(r);
+  }
+  if (ctx->preserve_caching_headers == kDontPreserveHeaders) {
+    // Don't cache html.  See mod_instaweb:instaweb_fix_headers_filter.
+    NgxCachingHeaders caching_headers(r);
+    ps_set_cache_control(r, string_piece_to_pool_string(
+        r->pool, caching_headers.GenerateDisabledCacheControl()));
+  }
+
+  // Pagespeed html doesn't need etags: it should never be cached.
+  ngx_http_clear_etag(r);
+
+  // An html page may change without the underlying file changing, because of
+  // how resources are included.  Pagespeed adds cache control headers for
+  // resources instead of using the last modified header.
+  ngx_http_clear_last_modified(r);
+
+  // Clear expires
+  if (r->headers_out.expires) {
+    r->headers_out.expires->hash = 0;
+    r->headers_out.expires = NULL;
+  }
+
+  return ngx_http_next_header_filter(r);
+}
+
+void ps_html_rewrite_fix_headers_filter_init() {
+  ngx_http_next_header_filter = ngx_http_top_header_filter;
+  ngx_http_top_header_filter = ps_html_rewrite_fix_headers_filter;
+}
+
+}  // namespace fix_headers
+
+using fix_headers::ps_html_rewrite_fix_headers_filter_init;
+
+
+// preaccess_handler should be at generic phase before try_files
+ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
+  ngx_http_core_main_conf_t* cmcf;
+  ngx_http_phase_handler_t* ph;
+  ngx_uint_t i;
+
+  cmcf = static_cast<ngx_http_core_main_conf_t*>(
+      ngx_http_get_module_main_conf(r, ngx_http_core_module));
+
+  ph = cmcf->phase_engine.handlers;
+
+  i = r->phase_handler;
+
+  // move handlers before try_files && content phase
+  // As of nginx 1.13.4 we will be right before the try_files module
+  #if (nginx_version < NGINX_1_13_4)
+  while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
+         ph[i + 1].checker != ngx_http_core_content_phase) {
+    ph[i] = ph[i + 1];
+    ph[i].next--;
+    i++;
+  }
+  #endif
+
+  // insert ps phase handler
+  ph[i].checker = ps_phase_handler;
+  ph[i].handler = NULL;
+  ph[i].next = i + 1;
+
+  // next preaccess handler
+  r->phase_handler--;
+  return NGX_DECLINED;
+}
+
+ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
+    ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
+    ngx_http_top_header_filter = ps_etag_header_filter;
+  }
+  return NGX_OK;
+}
+
+// Called before configuration.
+ngx_int_t ps_pre_init(ngx_conf_t* cf) {
+  // Setup an intervention setter for gzip configuration and check
+  // gzip configuration command signatures.
+  g_gzip_setter.Init(cf);
+  return NGX_OK;
+}
+
+ngx_int_t ps_init(ngx_conf_t* cf) {
+  // Only put register pagespeed code to run if there was a "pagespeed"
+  // configuration option set in the config file.  With "pagespeed off" we
+  // consider every request and choose not to do anything, while with no
+  // "pagespeed" directives we won't have any effect after nginx is done loading
+  // its configuration.
+
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
+
+  // The driver factory is on the main config and is non-NULL iff there is a
+  // pagespeed configuration option in the main config or a server block.  Note
+  // that if any server block has pagespeed 'on' then our header filter, body
+  // filter, and content handler will run in every server block.  This is ok,
+  // because they will notice that the server context is NULL and do nothing.
+  if (cfg_m->driver_factory != NULL) {
+    // The filter init order is important.
+    ps_in_place_filter_init();
+
+    ps_html_rewrite_fix_headers_filter_init();
+    ps_base_fetch::ps_base_fetch_filter_init();
+    ps_html_rewrite_filter_init();
+
+    ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
+        ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module));
+
+    int phase = NGX_HTTP_PREACCESS_PHASE;
+
+    // As of nginx 1.13.4, try_files has changed.
+    // https://github.com/nginx/nginx/commit/129b06dc5dfab7b4513a4f274b3778cd9b8a6a22
+#if (nginx_version >= NGINX_1_13_4)
+    phase = NGX_HTTP_PRECONTENT_PHASE;
+#endif
+
+    ngx_http_handler_pt* h = static_cast<ngx_http_handler_pt*>(
+        ngx_array_push(&cmcf->phases[phase].handlers));
+
+    if (h == NULL) {
+      return NGX_ERROR;
+    }
+    *h = ps_preaccess_handler;
+  }
+
+  return NGX_OK;
+}
+
+ngx_http_module_t ps_etag_filter_module = {
+  NULL,  // preconfiguration
+  ps_etag_filter_init,  // postconfiguration
+  NULL,
+  NULL,  // initialize main configuration
+  NULL,
+  NULL,
+  NULL,
+  NULL
+};
+
+ngx_http_module_t ps_module = {
+  ps_pre_init,  // preconfiguration
+  ps_init,  // postconfiguration
+
+  ps_create_main_conf,
+  NULL,  // initialize main configuration
+
+  ps_create_srv_conf,
+  ps_merge_srv_conf,
+
+  ps_create_loc_conf,
+  ps_merge_loc_conf
+};
+
+// called after configuration is complete, but before nginx starts forking
+ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
+
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1220
+  if (cfg_m == NULL) {
+    return NGX_OK;
+  }
+
+  ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
+  ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
+      cmcf->servers.elts);
+  ngx_uint_t s;
+
+  std::vector<SystemServerContext*> server_contexts;
+  // Iterate over all configured server{} blocks to collect the server contexts.
+  for (s = 0; s < cmcf->servers.nelts; s++) {
+    ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+        cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
+    if (cfg_s->server_context != NULL) {
+      server_contexts.push_back(cfg_s->server_context);
+    }
+  }
+
+  GoogleString error_message;
+  int error_index = -1;
+  Statistics* global_statistics = NULL;
+  cfg_m->driver_factory->PostConfig(
+      server_contexts, &error_message, &error_index, &global_statistics);
+  if (error_index != -1) {
+    server_contexts[error_index]->message_handler()->Message(
+        kError, "ngx_pagespeed is enabled. %s", error_message.c_str());
+    return NGX_ERROR;
+  }
+
+  if (!server_contexts.empty()) {
+    // TODO(oschaaf): this ignores sigpipe messages from memcached.
+    // however, it would be better to not have those signals generated
+    // in the first place, as suppressing them this way may interfere
+    // with other modules that actually are interested in these signals
+    ps_ignore_sigpipe();
+
+    // If no shared-mem statistics are enabled, then init using the default
+    // NullStatistics.
+    if (global_statistics == NULL) {
+      NgxRewriteDriverFactory::InitStats(cfg_m->driver_factory->statistics());
+    }
+
+    ngx_http_core_loc_conf_t* clcf = static_cast<ngx_http_core_loc_conf_t*>(
+        ngx_http_conf_get_module_loc_conf((*cscfp), ngx_http_core_module));
+
+    cfg_m->driver_factory->set_resolver(clcf->resolver);
+    cfg_m->driver_factory->set_resolver_timeout(clcf->resolver_timeout);
+
+    if (!cfg_m->driver_factory->CheckResolver()) {
+      cfg_m->handler->Message(
+          kError,
+          "UseNativeFetcher is on, please configure a resolver.");
+      return NGX_ERROR;
+    }
+
+    cfg_m->driver_factory->LoggingInit(cycle->log, true);
+    cfg_m->driver_factory->RootInit();
+  } else {
+    delete cfg_m->driver_factory;
+    cfg_m->driver_factory = NULL;
+    active_driver_factory = NULL;
+  }
+  return NGX_OK;
+}
+
+void ps_exit_child_process(ngx_cycle_t* cycle) {
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
+  NgxBaseFetch::Terminate();
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
+    cfg_m->driver_factory->ShutDown();
+  }
+}
+
+// Called when nginx forks worker processes.  No threads should be started
+// before this.
+ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
+  ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
+  if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
+    return NGX_OK;
+  }
+
+  if (!NgxBaseFetch::Initialize(cycle)) {
+    return NGX_ERROR;
+  }
+
+  // ChildInit() will initialise all ServerContexts, which we need to
+  // create ProxyFetchFactories below
+  cfg_m->driver_factory->LoggingInit(cycle->log, true);
+  cfg_m->driver_factory->ChildInit();
+
+  ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
+      ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
+  ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
+      cmcf->servers.elts);
+  ngx_uint_t s;
+
+  // Iterate over all configured server{} blocks, and find our context in it,
+  // so we can create and set a ProxyFetchFactory for it.
+  for (s = 0; s < cmcf->servers.nelts; s++) {
+    ps_srv_conf_t* cfg_s = static_cast<ps_srv_conf_t*>(
+        cscfp[s]->ctx->srv_conf[ngx_pagespeed.ctx_index]);
+    // Some server{} blocks may not have a ServerContext in that case we must
+    // not instantiate a ProxyFetchFactory.
+    if (cfg_s->server_context != NULL) {
+      cfg_s->proxy_fetch_factory = new ProxyFetchFactory(cfg_s->server_context);
+      ngx_http_core_loc_conf_t* clcf = static_cast<ngx_http_core_loc_conf_t*>(
+          cscfp[s]->ctx->loc_conf[ngx_http_core_module.ctx_index]);
+      cfg_m->driver_factory->SetServerContextMessageHandler(
+          cfg_s->server_context, clcf->error_log);
+    }
+  }
+
+  cfg_m->driver_factory->StartThreads();
+  return NGX_OK;
+}
+
+}  // namespace
+
+}  // namespace net_instaweb
+
+ngx_module_t ngx_pagespeed_etag_filter = {
+  NGX_MODULE_V1,
+  &net_instaweb::ps_etag_filter_module,
+  NULL,
+  NGX_HTTP_MODULE,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NGX_MODULE_V1_PADDING
+};
+
+ngx_module_t ngx_pagespeed = {
+  NGX_MODULE_V1,
+  &net_instaweb::ps_module,
+  net_instaweb::ps_commands,
+  NGX_HTTP_MODULE,
+  NULL,
+  net_instaweb::ps_init_module,
+  net_instaweb::ps_init_child_process,
+  NULL,
+  NULL,
+  net_instaweb::ps_exit_child_process,
+  NULL,
+  NGX_MODULE_V1_PADDING
+};

lib/ngx_pagespeed/src/ngx_pagespeed.h

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_PAGESPEED_H_
+#define NGX_PAGESPEED_H_
+
+// We might be compiled with syslog.h, which #defines LOG_INFO and LOG_WARNING
+// as ints.  But logging.h assumes they're usable as names, within their
+// namespace, so we need to #undef them before including logging.h
+#ifdef LOG_INFO
+#undef LOG_INFO
+#endif
+#ifdef LOG_WARNING
+#undef LOG_WARNING
+#endif
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "base/logging.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/http/response_headers.h"
+
+namespace net_instaweb {
+
+class GzipInflater;
+class NgxBaseFetch;
+class ProxyFetch;
+class RewriteDriver;
+class RequestHeaders;
+class ResponseHeaders;
+class InPlaceResourceRecorder;
+
+// Allocate chain links and buffers from the supplied pool, and copy over the
+// data from the string piece.  If the string piece is empty, return
+// NGX_DECLINED immediately unless send_last_buf.
+ngx_int_t string_piece_to_buffer_chain(
+    ngx_pool_t* pool, StringPiece sp,
+    ngx_chain_t** link_ptr, bool send_last_buf, bool send_flush);
+
+StringPiece str_to_string_piece(ngx_str_t s);
+
+// s1: ngx_str_t, s2: string literal
+// true if they're equal, false otherwise
+#define STR_EQ_LITERAL(s1, s2)          \
+    ((s1).len == (sizeof(s2)-1) &&      \
+     ngx_strncmp((s1).data, (s2), (sizeof(s2)-1)) == 0)
+
+// s1: ngx_str_t, s2: string literal
+// true if they're equal ignoring case, false otherwise
+#define STR_CASE_EQ_LITERAL(s1, s2)     \
+    ((s1).len == (sizeof(s2)-1) &&      \
+     ngx_strncasecmp((s1).data, (       \
+       reinterpret_cast<u_char*>(       \
+         const_cast<char*>(s2))),       \
+       (sizeof(s2)-1)) == 0)
+
+// Allocate memory out of the pool for the string piece, and copy the contents
+// over.  Returns NULL if we can't get memory.
+char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
+
+enum PreserveCachingHeaders {
+  kPreserveAllCachingHeaders,  // Cache-Control, ETag, Last-Modified, etc
+  kPreserveOnlyCacheControl,   // Only Cache-Control.
+  kDontPreserveHeaders,
+};
+
+typedef struct {
+  NgxBaseFetch* base_fetch;
+
+  ngx_http_request_t* r;
+
+  bool html_rewrite;
+  bool in_place;
+
+  PreserveCachingHeaders preserve_caching_headers;
+
+  // for html rewrite
+  ProxyFetch* proxy_fetch;
+  GzipInflater* inflater_;
+
+  // for in place resource
+  RewriteDriver* driver;
+  InPlaceResourceRecorder* recorder;
+  ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
+
+  // We need to remember if the upstream had headers_out->location set, because
+  // we should mirror that when we write it back. nginx may absolutify
+  // Location: headers that start with '/' without regarding X-Forwarded-Proto.
+  bool location_field_set;
+  bool psol_vary_accept_only;
+  bool follow_flushes;
+} ps_request_ctx_t;
+
+ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);
+
+void copy_request_headers_from_ngx(const ngx_http_request_t* r,
+                                   RequestHeaders* headers);
+
+void copy_response_headers_from_ngx(const ngx_http_request_t* r,
+                                    ResponseHeaders* headers);
+
+ngx_int_t copy_response_headers_to_ngx(
+    ngx_http_request_t* r,
+    const ResponseHeaders& pagespeed_headers,
+    PreserveCachingHeaders preserve_caching_headers);
+
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
+namespace ps_base_fetch {
+
+ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
+
+}  // namespace ps_base_fetch
+
+}  // namespace net_instaweb
+
+#endif  // NGX_PAGESPEED_H_

lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.cc

@@ -0,0 +1,288 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_rewrite_driver_factory.h"
+
+#include <cstdio>
+
+#include "log_message_handler.h"
+#include "ngx_message_handler.h"
+#include "ngx_rewrite_options.h"
+#include "ngx_server_context.h"
+#include "ngx_url_async_fetcher.h"
+
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
+#include "net/instaweb/http/public/wget_url_fetcher.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
+#include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/util/public/property_cache.h"
+#include "pagespeed/kernel/base/google_message_handler.h"
+#include "pagespeed/kernel/base/null_shared_mem.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/stdio_file_system.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/http/content_type.h"
+#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
+#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
+#include "pagespeed/kernel/thread/pthread_shared_mem.h"
+#include "pagespeed/kernel/thread/scheduler_thread.h"
+#include "pagespeed/kernel/thread/slow_worker.h"
+#include "pagespeed/system/in_place_resource_recorder.h"
+#include "pagespeed/system/serf_url_async_fetcher.h"
+#include "pagespeed/system/system_caches.h"
+#include "pagespeed/system/system_rewrite_options.h"
+
+namespace net_instaweb {
+
+class FileSystem;
+class Hasher;
+class MessageHandler;
+class Statistics;
+class Timer;
+class UrlAsyncFetcher;
+class UrlFetcher;
+class Writer;
+
+class SharedCircularBuffer;
+
+NgxRewriteDriverFactory::NgxRewriteDriverFactory(
+    const ProcessContext& process_context,
+    SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
+    : SystemRewriteDriverFactory(process_context, system_thread_system,
+        NULL /* default shared memory runtime */, hostname, port),
+      threads_started_(false),
+      ngx_message_handler_(
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+      ngx_html_parse_message_handler_(
+          new NgxMessageHandler(timer(), thread_system()->NewMutex())),
+      log_(NULL),
+      resolver_timeout_(NGX_CONF_UNSET_MSEC),
+      use_native_fetcher_(false),
+      // 100 Aligns to nginx's server-side default.
+      native_fetcher_max_keepalive_requests_(100),
+      ngx_shared_circular_buffer_(NULL),
+      hostname_(hostname.as_string()),
+      port_(port),
+      process_script_variables_mode_(ProcessScriptVariablesMode::kOff),
+      process_script_variables_set_(false),
+      shut_down_(false) {
+  InitializeDefaultOptions();
+  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
+  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
+      default_options());
+  system_options->set_file_cache_clean_inode_limit(500000);
+  system_options->set_avoid_renaming_introspective_javascript(true);
+  set_message_handler(ngx_message_handler_);
+  set_html_parse_message_handler(ngx_html_parse_message_handler_);
+}
+
+NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
+  ShutDown();
+  ngx_shared_circular_buffer_ = NULL;
+  STLDeleteElements(&uninitialized_server_contexts_);
+}
+
+Hasher* NgxRewriteDriverFactory::NewHasher() {
+  return new MD5Hasher;
+}
+
+UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
+    SystemRewriteOptions* config) {
+  if (use_native_fetcher_) {
+    NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
+        config->fetcher_proxy().c_str(),
+        log_,
+        resolver_timeout_,
+        config->blocking_fetch_timeout_ms(),
+        resolver_,
+        native_fetcher_max_keepalive_requests_,
+        thread_system(),
+        message_handler());
+    ngx_url_async_fetchers_.push_back(fetcher);
+    return fetcher;
+  } else {
+    return SystemRewriteDriverFactory::AllocateFetcher(config);
+  }
+}
+
+MessageHandler* NgxRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
+  return ngx_html_parse_message_handler_;
+}
+
+MessageHandler* NgxRewriteDriverFactory::DefaultMessageHandler() {
+  return ngx_message_handler_;
+}
+
+FileSystem* NgxRewriteDriverFactory::DefaultFileSystem() {
+  return new StdioFileSystem();
+}
+
+Timer* NgxRewriteDriverFactory::DefaultTimer() {
+  return new PosixTimer;
+}
+
+NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
+  CHECK(false);
+  return NULL;
+}
+
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
+  NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
+  options->SetRewriteLevel(RewriteOptions::kCoreFilters);
+  return options;
+}
+
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
+  return new NgxRewriteOptions(thread_system());
+}
+
+bool NgxRewriteDriverFactory::CheckResolver() {
+  if (use_native_fetcher_ && resolver_ == NULL) {
+    return false;
+  }
+  return true;
+}
+
+NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
+    StringPiece hostname, int port) {
+  NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
+  uninitialized_server_contexts_.insert(server_context);
+  return server_context;
+}
+
+ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
+  ServerContext* sc = new NgxServerContext(this, hostname_, port_);
+  InitStubDecodingServerContext(sc);
+  return sc;
+}
+
+ServerContext* NgxRewriteDriverFactory::NewServerContext() {
+  LOG(DFATAL) << "MakeNgxServerContext should be used instead";
+  return NULL;
+}
+
+void NgxRewriteDriverFactory::ShutDown() {
+  if (!shut_down_) {
+    shut_down_ = true;
+    SystemRewriteDriverFactory::ShutDown();
+  }
+}
+
+void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
+  ngx_message_handler_->set_buffer(NULL);
+  ngx_html_parse_message_handler_->set_buffer(NULL);
+  for (NgxMessageHandlerSet::iterator p =
+           server_context_message_handlers_.begin();
+       p != server_context_message_handlers_.end(); ++p) {
+    (*p)->set_buffer(NULL);
+  }
+  server_context_message_handlers_.clear();
+}
+
+void NgxRewriteDriverFactory::StartThreads() {
+  if (threads_started_) {
+    return;
+  }
+  // TODO(jefftk): use a native nginx timer instead of running our own thread.
+  // See issue #111.
+  SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
+  bool ok = thread->Start();
+  CHECK(ok) << "Unable to start scheduler thread";
+  defer_cleanup(thread->MakeDeleter());
+  threads_started_ = true;
+}
+
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
+void NgxRewriteDriverFactory::LoggingInit(
+    ngx_log_t* log, bool may_install_crash_handler) {
+  log_ = log;
+  net_instaweb::log_message_handler::Install(log);
+  if (may_install_crash_handler && install_crash_handler()) {
+    NgxMessageHandler::InstallCrashHandler(log);
+  }
+  ngx_message_handler_->set_log(log);
+  ngx_html_parse_message_handler_->set_log(log);
+}
+
+void NgxRewriteDriverFactory::SetCircularBuffer(
+    SharedCircularBuffer* buffer) {
+  ngx_shared_circular_buffer_ = buffer;
+  ngx_message_handler_->set_buffer(buffer);
+  ngx_html_parse_message_handler_->set_buffer(buffer);
+}
+
+void NgxRewriteDriverFactory::SetServerContextMessageHandler(
+    ServerContext* server_context, ngx_log_t* log) {
+  NgxMessageHandler* handler = new NgxMessageHandler(
+      timer(), thread_system()->NewMutex());
+  handler->set_log(log);
+  // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
+  // been raised from its default of 0.
+  handler->set_buffer(ngx_shared_circular_buffer_);
+  server_context_message_handlers_.insert(handler);
+  defer_cleanup(new Deleter<NgxMessageHandler>(handler));
+  server_context->set_message_handler(handler);
+}
+
+void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
+  // Init standard PSOL stats.
+  SystemRewriteDriverFactory::InitStats(statistics);
+  RewriteDriverFactory::InitStats(statistics);
+  RateController::InitStats(statistics);
+
+  // Init Ngx-specific stats.
+  NgxServerContext::InitStats(statistics);
+  InPlaceResourceRecorder::InitStats(statistics);
+}
+
+void NgxRewriteDriverFactory::PrepareForkedProcess(const char* name) {
+  ngx_pid = ngx_getpid();  // Needed for logging to have the right PIDs.
+  SystemRewriteDriverFactory::PrepareForkedProcess(name);
+}
+
+void NgxRewriteDriverFactory::NameProcess(const char* name) {
+  SystemRewriteDriverFactory::NameProcess(name);
+
+  // Superclass set status with prctl.  Nginx has a helper function for setting
+  // argv[0] as well, so let's use that.  We'll show up as:
+  //
+  //    nginx: pagespeed $name
+
+  char name_for_setproctitle[32];
+  snprintf(name_for_setproctitle, sizeof(name_for_setproctitle),
+           "pagespeed %s", name);
+  ngx_setproctitle(name_for_setproctitle);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_rewrite_driver_factory.h

@@ -0,0 +1,176 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_REWRITE_DRIVER_FACTORY_H_
+#define NGX_REWRITE_DRIVER_FACTORY_H_
+
+extern "C" {
+  #include <ngx_auto_config.h>
+#if (NGX_THREADS)
+  #include <ngx_thread.h>
+#endif
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+  #include <ngx_config.h>
+  #include <ngx_log.h>
+}
+
+#include <set>
+
+#include "pagespeed/kernel/base/md5_hasher.h"
+#include "pagespeed/kernel/base/scoped_ptr.h"
+#include "pagespeed/system/system_rewrite_driver_factory.h"
+
+namespace net_instaweb {
+
+class NgxMessageHandler;
+class NgxRewriteOptions;
+class NgxServerContext;
+class NgxUrlAsyncFetcher;
+class SharedCircularBuffer;
+class SharedMemRefererStatistics;
+class SlowWorker;
+class Statistics;
+class SystemThreadSystem;
+
+enum ProcessScriptVariablesMode {
+  kOff,
+  kLegacyRestricted,
+  kAll
+};
+
+class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
+ public:
+  // We take ownership of the thread system.
+  explicit NgxRewriteDriverFactory(
+      const ProcessContext& process_context,
+      SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
+  virtual ~NgxRewriteDriverFactory();
+  virtual Hasher* NewHasher();
+  virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
+  virtual MessageHandler* DefaultHtmlParseMessageHandler();
+  virtual MessageHandler* DefaultMessageHandler();
+  virtual FileSystem* DefaultFileSystem();
+  virtual Timer* DefaultTimer();
+  virtual NamedLockManager* DefaultLockManager();
+  // Create a new RewriteOptions.  In this implementation it will be an
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
+  virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
+  virtual ServerContext* NewDecodingServerContext();
+  // Check resolver configured or not.
+  bool CheckResolver();
+
+  // Initializes all the statistics objects created transitively by
+  // NgxRewriteDriverFactory, including nginx-specific and
+  // platform-independent statistics.
+  static void InitStats(Statistics* statistics);
+  NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
+  virtual ServerContext* NewServerContext();
+  virtual void ShutDown();
+
+  // Starts pagespeed threads if they've not been started already.  Must be
+  // called after the caller has finished any forking it intends to do.
+  void StartThreads();
+
+  void SetServerContextMessageHandler(ServerContext* server_context,
+                                      ngx_log_t* log);
+
+  NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
+
+  virtual void NonStaticInitStats(Statistics* statistics) {
+    InitStats(statistics);
+  }
+
+  void SetMainConf(NgxRewriteOptions* main_conf);
+
+  void set_resolver(ngx_resolver_t* resolver) {
+    resolver_ = resolver;
+  }
+  void set_resolver_timeout(ngx_msec_t resolver_timeout) {
+    resolver_timeout_ = resolver_timeout == NGX_CONF_UNSET_MSEC ?
+        1000 : resolver_timeout;
+  }
+  bool use_native_fetcher() {
+    return use_native_fetcher_;
+  }
+  void set_use_native_fetcher(bool x) {
+    use_native_fetcher_ = x;
+  }
+  int native_fetcher_max_keepalive_requests() {
+    return native_fetcher_max_keepalive_requests_;
+  }
+  void set_native_fetcher_max_keepalive_requests(int x) {
+    native_fetcher_max_keepalive_requests_ = x;
+  }
+  ProcessScriptVariablesMode process_script_variables() {
+    return process_script_variables_mode_;
+  }
+
+  void LoggingInit(ngx_log_t* log, bool may_install_crash_handler);
+
+  virtual void ShutDownMessageHandlers();
+
+  virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
+
+  bool SetProcessScriptVariables(ProcessScriptVariablesMode mode) {
+    if (!process_script_variables_set_) {
+      process_script_variables_mode_ = mode;
+      process_script_variables_set_ = true;
+      return true;
+    }
+    return false;
+  }
+
+  virtual void PrepareForkedProcess(const char* name);
+
+  virtual void NameProcess(const char* name);
+
+ private:
+  Timer* timer_;
+
+  bool threads_started_;
+  NgxMessageHandler* ngx_message_handler_;
+  NgxMessageHandler* ngx_html_parse_message_handler_;
+
+  std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
+  ngx_log_t* log_;
+  ngx_msec_t resolver_timeout_;
+  ngx_resolver_t* resolver_;
+  bool use_native_fetcher_;
+  int native_fetcher_max_keepalive_requests_;
+
+  typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
+  NgxMessageHandlerSet server_context_message_handlers_;
+
+  // Owned by the superclass.
+  // TODO(jefftk): merge the nginx and apache ways of doing this.
+  SharedCircularBuffer* ngx_shared_circular_buffer_;
+
+  GoogleString hostname_;
+  int port_;
+  ProcessScriptVariablesMode process_script_variables_mode_;
+  bool process_script_variables_set_;
+  bool shut_down_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_REWRITE_DRIVER_FACTORY_H_

lib/ngx_pagespeed/src/ngx_rewrite_options.cc

@@ -0,0 +1,555 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_rewrite_options.h"
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+}
+
+#include "ngx_pagespeed.h"
+#include "ngx_rewrite_driver_factory.h"
+
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/rewriter/public/file_load_policy.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/system/system_caches.h"
+
+namespace net_instaweb {
+
+namespace {
+
+const char kStatisticsPath[] = "StatisticsPath";
+const char kGlobalStatisticsPath[] = "GlobalStatisticsPath";
+const char kConsolePath[] = "ConsolePath";
+const char kMessagesPath[] = "MessagesPath";
+const char kAdminPath[] = "AdminPath";
+const char kGlobalAdminPath[] = "GlobalAdminPath";
+
+// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
+// indicates that they can not be set at the directory/location level. They set
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
+// TODO(oschaaf): this duplication is a short term solution.
+const char* const server_only_options[] = {
+  "FetcherTimeoutMs",
+  "FetchProxy",
+  "ForceCaching",
+  "GeneratedFilePrefix",
+  "ImgMaxRewritesAtOnce",
+  "InheritVHostConfig",
+  "InstallCrashHandler",
+  "MessageBufferSize",
+  "NumRewriteThreads",
+  "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
+  "TrackOriginalContentLength",
+  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
+  "BlockingRewriteRefererUrls",
+  "CreateSharedMemoryMetadataCache",
+  "LoadFromFile",
+  "LoadFromFileMatch",
+  "LoadFromFileRule",
+  "LoadFromFileRuleMatch",
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
+};
+
+// Options that can only be used in the main (http) option scope.
+const char* const main_only_options[] = {
+  "UseNativeFetcher",
+  "NativeFetcherMaxKeepaliveRequests"
+};
+
+}  // namespace
+
+RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
+
+NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
+                                     ThreadSystem* thread_system)
+    : SystemRewriteOptions(description, thread_system) {
+  Init();
+}
+
+NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
+    : SystemRewriteOptions(thread_system) {
+  Init();
+}
+
+void NgxRewriteOptions::Init() {
+  DCHECK(ngx_properties_ != NULL)
+      << "Call NgxRewriteOptions::Initialize() before construction";
+  clear_inherited_scripts_ = false;
+  InitializeOptions(ngx_properties_);
+}
+
+void NgxRewriteOptions::AddProperties() {
+  // Nginx-specific options.
+  add_ngx_option(
+      "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
+      kGlobalStatisticsPath, kProcessScopeStrict,
+      "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
+      "Set the console path. Ex: /pagespeed_console", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message",
+      false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin", false);
+  add_ngx_option(
+      "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
+      kProcessScopeStrict,
+      "Set the global admin path.  Ex: /pagespeed_global_admin",
+      false);
+
+  MergeSubclassProperties(ngx_properties_);
+
+  // Default properties are global but to set them the current API requires
+  // a RewriteOptions instance and we're in a static method.
+  NgxRewriteOptions dummy_config(NULL);
+  dummy_config.set_default_x_header_value(kModPagespeedVersion);
+}
+
+void NgxRewriteOptions::Initialize() {
+  if (Properties::Initialize(&ngx_properties_)) {
+    SystemRewriteOptions::Initialize();
+    AddProperties();
+  }
+}
+
+void NgxRewriteOptions::Terminate() {
+  if (Properties::Terminate(&ngx_properties_)) {
+    SystemRewriteOptions::Terminate();
+  }
+}
+
+bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
+                                    StringPiece compare_directive) {
+  return StringCaseEqual(config_directive, compare_directive);
+}
+
+RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
+    StringPiece option_name) {
+  ngx_uint_t i;
+  ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(main_only_options[i], option_name)) {
+      return kProcessScopeStrict;
+    }
+  }
+
+  size = sizeof(server_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(server_only_options[i], option_name)) {
+      return kServerScope;
+    }
+  }
+
+  // This could be made more efficient if RewriteOptions provided a map allowing
+  // access of options by their name. It's not too much of a worry at present
+  // since this is just during initialization.
+  for (OptionBaseVector::const_iterator it = all_options().begin();
+       it != all_options().end(); ++it) {
+    RewriteOptions::OptionBase* option = *it;
+    if (option->option_name() == option_name) {
+      // We treat kLegacyProcessScope as kProcessScopeStrict, failing to start
+      // if an option is out of place.
+      return option->scope() == kLegacyProcessScope ? kProcessScopeStrict
+                                                    : option->scope();
+    }
+  }
+  return kDirectoryScope;
+}
+
+RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
+    StringPiece directive, GoogleString* msg, MessageHandler* handler) {
+  EnabledEnum enabled;
+  if (!ParseFromString(directive, &enabled)) {
+    return RewriteOptions::kOptionNameUnknown;
+  }
+  if (enabled == RewriteOptions::kEnabledOff) {
+    // In ngx_pagespeed, for historical reasons, we treat "off" as "unplugged".
+    // Also, "off" is deprecated and people should be using "standby" or
+    // "unplugged" now depending on which sense they want.  See comment on
+    // RewriteOptions::EnabledEnum.
+    enabled = RewriteOptions::kEnabledUnplugged;
+  }
+  set_enabled(enabled);
+  return RewriteOptions::kOptionOk;
+}
+
+RewriteOptions::OptionSettingResult
+    NgxRewriteOptions::ParseAndSetOptionFromName1(
+        StringPiece name, StringPiece arg,
+        GoogleString* msg, MessageHandler* handler) {
+  // FileCachePath needs error checking.
+  if (StringCaseEqual(name, kFileCachePath)) {
+    if (!StringCaseStartsWith(arg, "/")) {
+      *msg = "must start with a slash";
+      return RewriteOptions::kOptionValueInvalid;
+    }
+  }
+
+  return SystemRewriteOptions::ParseAndSetOptionFromName1(
+      name, arg, msg, handler);
+}
+
+template <class DriverFactoryT>
+RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
+    StringPiece option_value,
+    DriverFactoryT* driver_factory,
+    void (DriverFactoryT::*set_option_method)(bool)) {
+  bool parsed_value;
+  if (StringCaseEqual(option_value, "on") ||
+      StringCaseEqual(option_value, "true")) {
+    parsed_value = true;
+  } else if (StringCaseEqual(option_value, "off") ||
+             StringCaseEqual(option_value, "false")) {
+    parsed_value = false;
+  } else {
+    return RewriteOptions::kOptionValueInvalid;
+  }
+
+  (driver_factory->*set_option_method)(parsed_value);
+  return RewriteOptions::kOptionOk;
+}
+
+namespace {
+
+const char* ps_error_string_for_option(
+    ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
+  GoogleString msg =
+      StrCat("\"", directive, "\" ", warning);
+  char* s = string_piece_to_pool_string(pool, msg);
+  if (s == NULL) {
+    return "failed to allocate memory";
+  }
+  return s;
+}
+
+}  // namespace
+
+// Very similar to apache/mod_instaweb::ParseDirective.
+const char* NgxRewriteOptions::ParseAndSetOptions(
+    StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
+    NgxRewriteDriverFactory* driver_factory,
+    RewriteOptions::OptionScope scope, ngx_conf_t* cf,
+    ProcessScriptVariablesMode script_mode) {
+  CHECK_GE(n_args, 1);
+
+  StringPiece directive = args[0];
+
+  // Remove initial "ModPagespeed" if there is one.
+  StringPiece mod_pagespeed("ModPagespeed");
+  if (StringCaseStartsWith(directive, mod_pagespeed)) {
+    directive.remove_prefix(mod_pagespeed.size());
+  }
+
+  if (GetOptionScope(directive) > scope) {
+    return ps_error_string_for_option(
+        pool, directive, "cannot be set at this scope.");
+  }
+
+  bool compile_scripts = false;
+
+  if (script_mode != ProcessScriptVariablesMode::kOff) {
+    // In the old mode we only allowed a few, so restrict to those.
+    compile_scripts =
+        StringCaseStartsWith(directive, "LoadFromFile") ||
+        StringCaseEqual(directive, "EnableFilters") ||
+        StringCaseEqual(directive, "DisableFilters") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") ||
+        StringCaseEqual(directive, "DownstreamCachePurgeMethod") ||
+        StringCaseEqual(directive,
+                        "DownstreamCacheRewrittenPercentageThreshold") ||
+        StringCaseEqual(directive, "ShardDomain");
+    // In the new behaviour we also allow scripting of query- and directory-
+    // scoped options.
+    compile_scripts |=
+        script_mode == ProcessScriptVariablesMode::kAll &&
+        (GetOptionScope(directive) <= RewriteOptions::kDirectoryScope ||
+         (StringCaseEqual(directive, "Allow") ||
+          StringCaseEqual(directive, "BlockingRewriteRefererUrls") ||
+          StringCaseEqual(directive, "Disallow") ||
+          StringCaseEqual(directive, "DistributableFilters") ||
+          StringCaseEqual(directive, "Domain") ||
+          StringCaseEqual(directive, "ExperimentVariable") ||
+          StringCaseEqual(directive, "ExperimentSpec") ||
+          StringCaseEqual(directive, "ForbidFilters") ||
+          StringCaseEqual(directive, "RetainComment") ||
+          StringCaseEqual(directive, "CustomFetchHeader") ||
+          StringCaseEqual(directive, "MapOriginDomain") ||
+          StringCaseEqual(directive, "MapProxyDomain") ||
+          StringCaseEqual(directive, "MapRewriteDomain") ||
+          StringCaseEqual(directive, "UrlValuedAttribute") ||
+          StringCaseEqual(directive, "Library")));
+  }
+
+  ScriptLine* script_line;
+  script_line = NULL;
+
+  if (n_args == 1 && StringCaseEqual(directive, "ClearInheritedScripts")) {
+    clear_inherited_scripts_ = true;
+    return NGX_CONF_OK;
+  }
+
+  if (compile_scripts) {
+    CHECK(cf != NULL);
+    int i;
+    // Skip the first arg which is always 'pagespeed'
+    for (i = 1; i < n_args; i++) {
+      ngx_str_t script_source;
+
+      script_source.len = args[i].as_string().length();
+      std::string tmp = args[i].as_string();
+      script_source.data = reinterpret_cast<u_char*>(
+          const_cast<char*>(tmp.c_str()));
+
+      if (ngx_http_script_variables_count(&script_source) > 0) {
+        ngx_http_script_compile_t* sc =
+            reinterpret_cast<ngx_http_script_compile_t*>(
+                ngx_pcalloc(cf->pool, sizeof(ngx_http_script_compile_t)));
+        sc->cf = cf;
+        sc->source = &script_source;
+        sc->lengths = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->values = reinterpret_cast<ngx_array_t**>(
+            ngx_pcalloc(cf->pool, sizeof(ngx_array_t*)));
+        sc->variables = 1;
+        sc->complete_lengths = 1;
+        sc->complete_values = 1;
+        if (ngx_http_script_compile(sc) != NGX_OK) {
+          return ps_error_string_for_option(
+              pool, directive, "Failed to compile script variables");
+        } else {
+          if (script_line == NULL) {
+            script_line = new ScriptLine(args, n_args, scope);
+          }
+          script_line->AddScriptAndArgIndex(sc, i);
+        }
+      }
+    }
+
+    if (script_line != NULL) {
+      script_lines_.push_back(RefCountedPtr<ScriptLine>(script_line));
+      // We have found script variables in the current configuration line, and
+      // prepared the associated rewriteoptions for that.
+      // We will defer parsing, validation and processing of this line to
+      // request time. That means we are done handling this configuration line.
+      return NGX_CONF_OK;
+    }
+  }
+
+  GoogleString msg;
+  OptionSettingResult result;
+  if (n_args == 1) {
+    result = ParseAndSetOptions0(directive, &msg, handler);
+  } else if (n_args == 2) {
+    StringPiece arg = args[1];
+    if (IsDirective(directive, "UseNativeFetcher")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_use_native_fetcher);
+    } else if (IsDirective(directive, "NativeFetcherMaxKeepaliveRequests")) {
+      int max_keepalive_requests;
+      if (StringToInt(arg, &max_keepalive_requests) &&
+          max_keepalive_requests > 0) {
+        driver_factory->set_native_fetcher_max_keepalive_requests(
+            max_keepalive_requests);
+        result = RewriteOptions::kOptionOk;
+      } else {
+        result = RewriteOptions::kOptionValueInvalid;
+      }
+    } else if (StringCaseEqual("ProcessScriptVariables", args[0])) {
+      if (scope == RewriteOptions::kProcessScopeStrict) {
+        ProcessScriptVariablesMode mode;
+        if (StringCaseEqual(arg, "all")) {
+          mode = ProcessScriptVariablesMode::kAll;
+        } else if (StringCaseEqual(arg, "on")) {
+          mode = ProcessScriptVariablesMode::kLegacyRestricted;
+        } else if (StringCaseEqual(arg, "off")) {
+          mode = ProcessScriptVariablesMode::kOff;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: invalid value");
+        }
+        if (driver_factory->SetProcessScriptVariables(mode)) {
+          result = RewriteOptions::kOptionOk;
+        } else {
+          return const_cast<char*>(
+              "pagespeed ProcessScriptVariables: can only be set once");
+        }
+      } else {
+        return const_cast<char*>(
+            "ProcessScriptVariables is only allowed at the top level");
+      }
+    } else {
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
+      if (result == RewriteOptions::kOptionNameUnknown) {
+        result = driver_factory->ParseAndSetOption1(
+            directive,
+            arg,
+            scope >= RewriteOptions::kLegacyProcessScope,
+            &msg,
+            handler);
+      }
+    }
+  } else if (n_args == 3) {
+    result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+                                        &msg, handler);
+    if (result == RewriteOptions::kOptionNameUnknown) {
+      result = driver_factory->ParseAndSetOption2(
+          directive,
+          args[1],
+          args[2],
+          scope >= RewriteOptions::kLegacyProcessScope,
+          &msg,
+          handler);
+    }
+  } else if (n_args == 4) {
+    result = ParseAndSetOptionFromName3(
+        directive, args[1], args[2], args[3], &msg, handler);
+  } else {
+    result = RewriteOptions::kOptionNameUnknown;
+  }
+
+  switch (result) {
+    case RewriteOptions::kOptionOk:
+      return NGX_CONF_OK;
+    case RewriteOptions::kOptionNameUnknown:
+      return ps_error_string_for_option(
+          pool, directive, "not recognized or too many arguments");
+    case RewriteOptions::kOptionValueInvalid: {
+      GoogleString full_directive;
+      for (int i = 0 ; i < n_args ; i++) {
+        StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
+      }
+      return ps_error_string_for_option(pool, full_directive, msg);
+    }
+  }
+
+  CHECK(false);
+  return NULL;
+}
+
+// Execute all entries in the script_lines vector, and hand the result off to
+// ParseAndSetOptions to obtain the final option values.
+bool NgxRewriteOptions::ExecuteScriptVariables(
+    ngx_http_request_t* r, MessageHandler* handler,
+    NgxRewriteDriverFactory* driver_factory) {
+  bool script_error = false;
+
+  if (script_lines_.size() > 0) {
+    std::vector<RefCountedPtr<ScriptLine> >::iterator it;
+    for (it = script_lines_.begin() ; it != script_lines_.end(); ++it) {
+      ScriptLine* script_line = it->get();
+      StringPiece args[NGX_PAGESPEED_MAX_ARGS];
+      std::vector<ScriptArgIndex*>::iterator cs_it;
+      int i;
+
+      for (i = 0; i < script_line->n_args(); i++) {
+        args[i] = script_line->args()[i];
+      }
+
+      for (cs_it = script_line->data().begin();
+           cs_it != script_line->data().end(); cs_it++) {
+        ngx_http_script_compile_t* script;
+        ngx_array_t* values;
+        ngx_array_t* lengths;
+        ngx_str_t value;
+
+        script = (*cs_it)->script();
+        lengths = *script->lengths;
+        values = *script->values;
+
+        if (ngx_http_script_run(r, &value, lengths->elts, 0, values->elts)
+            == NULL) {
+          handler->Message(kError, "ngx_http_script_run error");
+          script_error = true;
+          break;
+        } else  {
+          args[(*cs_it)->index()] = str_to_string_piece(value);
+        }
+      }
+
+      const char* status = ParseAndSetOptions(args, script_line->n_args(),
+          r->pool, handler, driver_factory, script_line->scope(), NULL /*cf*/,
+          ProcessScriptVariablesMode::kOff);
+
+      if (status != NULL) {
+        script_error = true;
+        handler->Message(kWarning,
+            "Error setting option value from script: '%s'", status);
+        break;
+      }
+    }
+  }
+
+  if (script_error) {
+    handler->Message(kWarning,
+        "Script error(s) in configuration, disabling optimization");
+    set_enabled(RewriteOptions::kEnabledOff);
+    return false;
+  }
+
+  return true;
+}
+
+void NgxRewriteOptions::CopyScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_ = script_lines_;
+}
+
+void NgxRewriteOptions::AppendScriptLinesTo(
+    NgxRewriteOptions* destination) const {
+  destination->script_lines_.insert(destination->script_lines_.end(),
+                                    script_lines_.begin(), script_lines_.end());
+}
+
+NgxRewriteOptions* NgxRewriteOptions::Clone() const {
+  NgxRewriteOptions* options = new NgxRewriteOptions(
+      StrCat("cloned from ", description()), thread_system());
+  this->CopyScriptLinesTo(options);
+  options->Merge(*this);
+  return options;
+}
+
+const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
+    const RewriteOptions* instance) {
+  return dynamic_cast<const NgxRewriteOptions*>(instance);
+}
+
+NgxRewriteOptions* NgxRewriteOptions::DynamicCast(RewriteOptions* instance) {
+  return dynamic_cast<NgxRewriteOptions*>(instance);
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_rewrite_options.h

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// Manage configuration for pagespeed.  Compare to ApacheConfig.
+
+#ifndef NGX_REWRITE_OPTIONS_H_
+#define NGX_REWRITE_OPTIONS_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+  #include <ngx_http.h>
+}
+
+#include <vector>
+
+#include "ngx_rewrite_driver_factory.h"
+
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/ref_counted_ptr.h"
+#include "pagespeed/kernel/base/stl_util.h"          // for STLDeleteElements
+#include "pagespeed/system/system_rewrite_options.h"
+
+#define NGX_PAGESPEED_MAX_ARGS 10
+
+namespace net_instaweb {
+
+class NgxRewriteDriverFactory;
+
+class ScriptArgIndex {
+ public:
+  explicit ScriptArgIndex(ngx_http_script_compile_t* script, int index)
+    : script_(script), index_(index) {
+      CHECK(script != NULL);
+      CHECK(index > 0 && index < NGX_PAGESPEED_MAX_ARGS);
+  }
+
+  virtual ~ScriptArgIndex() {}
+
+  ngx_http_script_compile_t* script() { return script_; }
+  int index() { return index_; }
+
+ private:
+  // Not owned.
+  ngx_http_script_compile_t* script_;
+  int index_;
+};
+
+// Refcounted, because the ScriptArgIndexes inside data_ can be shared between
+// different rewriteoptions.
+class ScriptLine : public RefCounted<ScriptLine> {
+ public:
+  explicit ScriptLine(StringPiece* args, int n_args,
+                      RewriteOptions::OptionScope scope)
+    : n_args_(n_args),
+      scope_(scope) {
+
+      for (int i = 0; i < n_args; i++) {
+        args_[i] = args[i];
+      }
+  }
+
+  virtual ~ScriptLine() {
+    STLDeleteElements(&data_);
+    data_.clear();
+  }
+
+  void AddScriptAndArgIndex(ngx_http_script_compile_t* script,
+                            int script_index) {
+    CHECK(script != NULL);
+    CHECK(script_index <  NGX_PAGESPEED_MAX_ARGS);
+    data_.push_back(new ScriptArgIndex(script, script_index));
+  }
+
+  int n_args() { return n_args_;}
+  StringPiece* args() { return args_;}
+  RewriteOptions::OptionScope scope() { return scope_; }
+  std::vector<ScriptArgIndex*>& data() {
+    return data_;
+  }
+
+ private:
+  StringPiece args_[NGX_PAGESPEED_MAX_ARGS];
+  int n_args_;
+  RewriteOptions::OptionScope scope_;
+  std::vector<ScriptArgIndex*> data_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScriptLine);
+};
+
+class NgxRewriteOptions : public SystemRewriteOptions {
+ public:
+  // See rewrite_options::Initialize and ::Terminate
+  static void Initialize();
+  static void Terminate();
+
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
+  explicit NgxRewriteOptions(ThreadSystem* thread_system);
+  virtual ~NgxRewriteOptions() { }
+
+  // args is an array of n_args StringPieces together representing a directive.
+  // For example:
+  //   ["RewriteLevel", "PassThrough"]
+  // or
+  //   ["EnableFilters", "combine_css,extend_cache,rewrite_images"]
+  // or
+  //   ["ShardDomain", "example.com", "s1.example.com,s2.example.com"]
+  // Apply the directive, returning NGX_CONF_OK on success or an error message
+  // on failure.
+  //
+  // pool is a memory pool for allocating error strings.
+  // cf is only required when compile_scripts is true
+  // when compile_scripts is true, the rewrite_options will be prepared
+  // for replacing any script $variables encountered in args. when false,
+  // script variables will be substituted using the prepared rewrite options.
+  const char* ParseAndSetOptions(
+      StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope,
+      ngx_conf_t* cf, ProcessScriptVariablesMode script_mode);
+  bool ExecuteScriptVariables(
+      ngx_http_request_t* r, MessageHandler* handler,
+      NgxRewriteDriverFactory* driver_factory);
+  void CopyScriptLinesTo(NgxRewriteOptions* destination) const;
+  void AppendScriptLinesTo(NgxRewriteOptions* destination) const;
+
+  // Make an identical copy of these options and return it.
+  virtual NgxRewriteOptions* Clone() const;
+
+  // Returns a suitably down cast version of 'instance' if it is an instance
+  // of this class, NULL if not.
+  static const NgxRewriteOptions* DynamicCast(const RewriteOptions* instance);
+  static NgxRewriteOptions* DynamicCast(RewriteOptions* instance);
+
+  const GoogleString& statistics_path() const {
+    return statistics_path_.value();
+  }
+  const GoogleString& global_statistics_path() const {
+    return global_statistics_path_.value();
+  }
+  const GoogleString& console_path() const {
+    return console_path_.value();
+  }
+  const GoogleString& messages_path() const {
+    return messages_path_.value();
+  }
+  const GoogleString& admin_path() const {
+    return admin_path_.value();
+  }
+  const GoogleString& global_admin_path() const {
+    return global_admin_path_.value();
+  }
+  const std::vector<RefCountedPtr<ScriptLine> >& script_lines() const {
+    return script_lines_;
+  }
+  const bool& clear_inherited_scripts() const {
+    return clear_inherited_scripts_;
+  }
+
+ private:
+  // Helper methods for ParseAndSetOptions().  Each can:
+  //  - return kOptionNameUnknown and not set msg:
+  //    - directive not handled; continue on with other possible
+  //      interpretations.
+  //  - return kOptionOk and not set msg:
+  //    - directive handled, all's well.
+  //  - return kOptionValueInvalid and set msg:
+  //    - directive handled with an error; return the error to the user.
+  //
+  // msg will be shown to the user on kOptionValueInvalid.  While it would be
+  // nice to always use msg and never use the MessageHandler, some option
+  // parsing code in RewriteOptions expects to write to a MessageHandler.  If
+  // that happens we put a summary on msg so the user sees something, and the
+  // detailed message goes to their log via handler.
+  OptionSettingResult ParseAndSetOptions0(
+      StringPiece directive, GoogleString* msg, MessageHandler* handler);
+
+  virtual OptionSettingResult ParseAndSetOptionFromName1(
+      StringPiece name, StringPiece arg,
+      GoogleString* msg, MessageHandler* handler);
+
+  // We may want to override 2- and 3-argument versions as well in the future,
+  // but they are not needed yet.
+
+  // Keeps the properties added by this subclass.  These are merged into
+  // RewriteOptions::all_properties_ during Initialize().
+  //
+  // RewriteOptions uses static initialization to reduce memory usage and
+  // construction time.  All NgxRewriteOptions instances will have the same
+  // Properties, so we can build the list when we initialize the first one.
+  static Properties* ngx_properties_;
+  static void AddProperties();
+  void Init();
+
+  // Add an option to ngx_properties_
+  template<class OptionClass>
+  static void add_ngx_option(typename OptionClass::ValueType default_value,
+                             OptionClass NgxRewriteOptions::*offset,
+                             const char* id,
+                             StringPiece option_name,
+                             OptionScope scope,
+                             const char* help,
+                             bool safe_to_print) {
+    AddProperty(default_value, offset, id, option_name, scope, help,
+                safe_to_print, ngx_properties_);
+  }
+
+  Option<GoogleString> statistics_path_;
+  Option<GoogleString> global_statistics_path_;
+  Option<GoogleString> console_path_;
+  Option<GoogleString> messages_path_;
+  Option<GoogleString> admin_path_;
+  Option<GoogleString> global_admin_path_;
+
+  bool clear_inherited_scripts_;
+  std::vector<RefCountedPtr<ScriptLine> > script_lines_;
+
+  // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
+  // ignoring case.
+  bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
+
+  // Returns a given option's scope.
+  RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
+
+  // TODO(jefftk): support fetch proxy in server and location blocks.
+
+  DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_REWRITE_OPTIONS_H_

lib/ngx_pagespeed/src/ngx_server_context.cc

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_server_context.h"
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "ngx_pagespeed.h"
+#include "ngx_message_handler.h"
+#include "ngx_rewrite_driver_factory.h"
+#include "ngx_rewrite_options.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "pagespeed/system/add_headers_fetcher.h"
+#include "pagespeed/system/loopback_route_fetcher.h"
+#include "pagespeed/system/system_request_context.h"
+
+namespace net_instaweb {
+
+NgxServerContext::NgxServerContext(
+    NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
+    : SystemServerContext(factory, hostname, port),
+      ngx_http2_variable_index_(NGX_ERROR) {
+}
+
+NgxServerContext::~NgxServerContext() { }
+
+NgxRewriteOptions* NgxServerContext::config() {
+  return NgxRewriteOptions::DynamicCast(global_options());
+}
+
+SystemRequestContext* NgxServerContext::NewRequestContext(
+    ngx_http_request_t* r) {
+  // Based on ngx_http_variable_server_port.
+  bool port_set = false;
+  int local_port = 0;
+#if (NGX_HAVE_INET6)
+  if (r->connection->local_sockaddr->sa_family == AF_INET6) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
+        r->connection->local_sockaddr)->sin6_port);
+    port_set = true;
+  }
+#endif
+  if (!port_set) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in*>(
+        r->connection->local_sockaddr)->sin_port);
+  }
+
+  ngx_str_t local_ip;
+  u_char addr[NGX_SOCKADDR_STRLEN];
+  local_ip.len = NGX_SOCKADDR_STRLEN;
+  local_ip.data = addr;
+  ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &local_ip, 0);
+  if (rc != NGX_OK) {
+    local_ip.len = 0;
+  }
+
+  SystemRequestContext* ctx = new SystemRequestContext(
+      thread_system()->NewMutex(), timer(),
+      ps_determine_host(r), local_port, str_to_string_piece(local_ip));
+
+  // See if http2 is in use.
+  if (ngx_http2_variable_index_ >= 0) {
+    ngx_http_variable_value_t* val =
+        ngx_http_get_indexed_variable(r, ngx_http2_variable_index_);
+    if (val != NULL && val->valid) {
+      StringPiece str_val(reinterpret_cast<char*>(val->data), val->len);
+      if (str_val == "h2" || str_val == "h2c") {
+        ctx->set_using_http2(true);
+      }
+    }
+  }
+
+  return ctx;
+}
+
+GoogleString NgxServerContext::FormatOption(StringPiece option_name,
+                                            StringPiece args) {
+  return StrCat("pagespeed ", option_name, " ", args, ";");
+}
+
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_server_context.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+// Manage pagespeed state across requests.  Compare to ApacheResourceManager.
+
+#ifndef NGX_SERVER_CONTEXT_H_
+#define NGX_SERVER_CONTEXT_H_
+
+#include "ngx_message_handler.h"
+#include "pagespeed/system/system_server_context.h"
+
+extern "C" {
+#include <ngx_http.h>
+}
+
+namespace net_instaweb {
+
+class NgxRewriteDriverFactory;
+class NgxRewriteOptions;
+class SystemRequestContext;
+
+class NgxServerContext : public SystemServerContext {
+ public:
+  NgxServerContext(
+      NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
+  virtual ~NgxServerContext();
+
+  // We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
+  // set_trusted_input() on any ProxyFetches we use to transform internal HTML.
+  virtual bool ProxiesHtml() const { return false; }
+
+  // Call only when you need an NgxRewriteOptions.  If you don't need
+  // nginx-specific behavior, call global_options() instead which doesn't
+  // downcast.
+  NgxRewriteOptions* config();
+
+  NgxRewriteDriverFactory* ngx_rewrite_driver_factory() { return ngx_factory_; }
+  SystemRequestContext* NewRequestContext(ngx_http_request_t* r);
+
+  NgxMessageHandler* ngx_message_handler() {
+    return dynamic_cast<NgxMessageHandler*>(message_handler());
+  }
+
+  virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);
+
+  void set_ngx_http2_variable_index(ngx_int_t idx) {
+    ngx_http2_variable_index_ = idx;
+  }
+
+  ngx_int_t ngx_http2_variable_index() const {
+    return ngx_http2_variable_index_;
+  }
+
+ private:
+  NgxRewriteDriverFactory* ngx_factory_;
+  // what index the "http2" var is, or NGX_ERROR.
+  ngx_int_t ngx_http2_variable_index_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_SERVER_CONTEXT_H_

lib/ngx_pagespeed/src/ngx_url_async_fetcher.cc

@@ -0,0 +1,299 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com (Junmin Xiong)
+
+extern "C" {
+  #include <ngx_http.h>
+  #include <ngx_core.h>
+}
+
+#include "ngx_url_async_fetcher.h"
+#include "ngx_fetch.h"
+
+#include <vector>
+#include <algorithm>
+#include <string>
+#include <list>
+#include <map>
+#include <set>
+
+#include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/condvar.h"
+#include "pagespeed/kernel/base/message_handler.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/pool_element.h"
+#include "pagespeed/kernel/base/statistics.h"
+#include "pagespeed/kernel/base/string_util.h"
+#include "pagespeed/kernel/base/thread_system.h"
+#include "pagespeed/kernel/base/timer.h"
+#include "pagespeed/kernel/base/writer.h"
+#include "pagespeed/kernel/http/request_headers.h"
+#include "pagespeed/kernel/http/response_headers.h"
+#include "pagespeed/kernel/http/response_headers_parser.h"
+
+namespace net_instaweb {
+
+  NgxUrlAsyncFetcher::NgxUrlAsyncFetcher(const char* proxy,
+                                         ngx_log_t* log,
+                                         ngx_msec_t resolver_timeout,
+                                         ngx_msec_t fetch_timeout,
+                                         ngx_resolver_t* resolver,
+                                         int max_keepalive_requests,
+                                         ThreadSystem* thread_system,
+                                         MessageHandler* handler)
+    : fetchers_count_(0),
+      shutdown_(false),
+      track_original_content_length_(false),
+      byte_count_(0),
+      thread_system_(thread_system),
+      message_handler_(handler),
+      mutex_(NULL),
+      max_keepalive_requests_(max_keepalive_requests),
+      event_connection_(NULL) {
+    resolver_timeout_ = resolver_timeout;
+    fetch_timeout_ = fetch_timeout;
+    ngx_memzero(&proxy_, sizeof(proxy_));
+    if (proxy != NULL && *proxy != '\0') {
+      proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
+      proxy_.url.len = ngx_strlen(proxy);
+    }
+    mutex_ = thread_system_->NewMutex();
+    log_ = log;
+    pool_ = NULL;
+    resolver_ = resolver;
+    // If init fails, set shutdown_ so no fetches will be attempted.
+    if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
+      shutdown_ = true;
+      message_handler_->Message(
+          kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
+    }
+  }
+
+  NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
+    DCHECK(shutdown_)  << "Shut down before destructing NgxUrlAsyncFetcher.";
+    message_handler_->Message(
+        kInfo,
+        "Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
+        ApproximateNumActiveFetches());
+
+    CancelActiveFetches();
+    active_fetches_.DeleteAll();
+    NgxConnection::Terminate();
+
+    if (pool_ != NULL) {
+      ngx_destroy_pool(pool_);
+      pool_ = NULL;
+    }
+    if (mutex_ != NULL) {
+      delete mutex_;
+      mutex_ = NULL;
+    }
+  }
+
+
+  bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
+    size_t scheme_offset;
+    u_short port;
+    if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                     const_cast<char*>("http://")), 7) == 0) {
+      scheme_offset = 7;
+      port = 80;
+    } else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                    const_cast<char*>("https://")), 8) == 0) {
+      scheme_offset = 8;
+      port = 443;
+    } else {
+      scheme_offset = 0;
+      port = 80;
+    }
+
+    url->url.data += scheme_offset;
+    url->url.len -= scheme_offset;
+    url->default_port = port;
+    // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
+    url->no_resolve = 0;
+    url->uri_part = 1;
+
+    if (ngx_parse_url(pool, url) == NGX_OK) {
+      return true;
+    }
+    return false;
+  }
+
+  // If there are still active requests, cancel them.
+  void NgxUrlAsyncFetcher::CancelActiveFetches() {
+    // TODO(oschaaf): this seems tricky, this may end up calling
+    // FetchComplete, modifying the active fetches while we are looping
+    // it
+    for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
+        e = active_fetches_.end(); p != e; ++p) {
+      NgxFetch* fetch = *p;
+      fetch->CallbackDone(false);
+    }
+  }
+
+  // Create the pool for fetcher, create the pipe, add the read event for main
+  // thread. It should be called in the worker process.
+  bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
+    log_ = cycle->log;
+    CHECK(event_connection_ == NULL) << "event connection already set";
+    event_connection_ = new NgxEventConnection(ReadCallback);
+    if (!event_connection_->Init(cycle)) {
+      return false;
+    }
+    if (pool_ == NULL) {
+      pool_ = ngx_create_pool(4096, log_);
+      if (pool_ == NULL) {
+        ngx_log_error(NGX_LOG_ERR, log_, 0,
+            "NgxUrlAsyncFetcher::Init create pool failed");
+        return false;
+      }
+    }
+
+    if (proxy_.url.len == 0) {
+      return true;
+    }
+
+    // TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
+    // up when parsing the url failed?
+    if (!ParseUrl(&proxy_, pool_)) {
+      ngx_log_error(NGX_LOG_ERR, log_, 0,
+          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
+      return false;
+    }
+    return true;
+  }
+
+  void NgxUrlAsyncFetcher::ShutDown() {
+    shutdown_ = true;
+    if (!pending_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
+           e = pending_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      pending_fetches_.DeleteAll();
+    }
+
+    if (!active_fetches_.empty()) {
+      for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
+           e = active_fetches_.end(); p != e; p++) {
+        NgxFetch* fetch = *p;
+        fetch->CallbackDone(false);
+      }
+      active_fetches_.Clear();
+    }
+    if (event_connection_ != NULL) {
+      event_connection_->Shutdown();
+      delete event_connection_;
+      event_connection_ = NULL;
+    }
+  }
+
+  // It's called in the rewrite thread. All the fetches are started at
+  // this function. It will notify the main thread to start the fetch job.
+  void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
+                                 MessageHandler* message_handler,
+                                 AsyncFetch* async_fetch) {
+    // Don't accept new fetches when shut down. This flow is also entered when
+    // we did not initialize properly in ::Init().
+    if (shutdown_) {
+      async_fetch->Done(false);
+      return;
+    }
+    async_fetch = EnableInflation(async_fetch);
+    NgxFetch* fetch = new NgxFetch(url, async_fetch,
+          message_handler, log_);
+    ScopedMutex lock(mutex_);
+    pending_fetches_.Add(fetch);
+
+    // TODO(oschaaf): thread safety on written vs shutdown.
+    // It is possible that shutdown() is called after writing an event? In that
+    // case, this could (rarely) fail when it shouldn't.
+    bool written = event_connection_->WriteEvent(this);
+    CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
+  }
+
+  // This is the read event which is called in the main thread.
+  // It will do the real work. Add the work event and start the fetch.
+  void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
+    std::vector<NgxFetch*> to_start;
+    NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
+      data.sender);
+
+    fetcher->mutex_->Lock();
+    fetcher->completed_fetches_.DeleteAll();
+
+    for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
+             e = fetcher->pending_fetches_.end(); p != e; p++) {
+      NgxFetch* fetch = *p;
+      to_start.push_back(fetch);
+    }
+
+    fetcher->pending_fetches_.Clear();
+    fetcher->mutex_->Unlock();
+
+    for (size_t i = 0; i < to_start.size(); i++) {
+      fetcher->StartFetch(to_start[i]);
+    }
+
+    return;
+  }
+
+  // TODO(oschaaf): return value is ignored.
+  bool NgxUrlAsyncFetcher::StartFetch(NgxFetch* fetch) {
+    mutex_->Lock();
+    active_fetches_.Add(fetch);
+    fetchers_count_++;
+    mutex_->Unlock();
+
+    // Don't initiate the fetch when we are shutting down
+    if (shutdown_) {
+      fetch->CallbackDone(false);
+      return false;
+    }
+
+    bool started = fetch->Start(this);
+
+    if (!started) {
+      message_handler_->Message(kWarning, "Fetch failed to start: %s",
+                                fetch->str_url());
+      fetch->CallbackDone(false);
+    }
+
+    return started;
+  }
+
+  void NgxUrlAsyncFetcher::FetchComplete(NgxFetch* fetch) {
+    ScopedMutex lock(mutex_);
+    byte_count_ += fetch->bytes_received();
+    fetchers_count_--;
+    active_fetches_.Remove(fetch);
+    completed_fetches_.Add(fetch);
+  }
+
+  void NgxUrlAsyncFetcher::PrintActiveFetches(MessageHandler* handler) const {
+    for (NgxFetchPool::const_iterator p = active_fetches_.begin(),
+        e = active_fetches_.end(); p != e; ++p) {
+      NgxFetch* fetch = *p;
+      handler->Message(kInfo, "Active fetch: %s", fetch->str_url());
+    }
+  }
+}  // namespace net_instaweb

lib/ngx_pagespeed/src/ngx_url_async_fetcher.h

@@ -0,0 +1,154 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: x.dinic@gmail.com(Junmin Xiong)
+//
+// Fetch the resources asynchronously in Nginx. The fetcher is called in
+// the rewrite thread.
+//
+// It can communicate with Nginx by pipe. One pipe for one fetcher.
+// When new url fetch comes, Fetcher will add it to the pending queue and
+// notify the Nginx thread to start the Fetch event. All the events are hooked
+// in the main thread's epoll structure.
+
+#ifndef NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
+#define NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_
+
+extern "C" {
+  #include <ngx_config.h>
+  #include <ngx_core.h>
+}
+
+#include <vector>
+
+#include "ngx_event_connection.h"
+
+#include "net/instaweb/http/public/url_async_fetcher.h"
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/base/pool.h"
+#include "pagespeed/kernel/base/string.h"
+#include "pagespeed/kernel/base/thread_system.h"
+
+
+namespace net_instaweb {
+
+class AsyncFetch;
+class MessageHandler;
+class Statistics;
+class NgxFetch;
+class Variable;
+
+class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
+ public:
+  NgxUrlAsyncFetcher(
+      const char* proxy, ngx_log_t* log, ngx_msec_t resolver_timeout,
+      ngx_msec_t fetch_timeout, ngx_resolver_t* resolver,
+      int max_keepalive_requests, ThreadSystem* thread_system,
+      MessageHandler* handler);
+
+  ~NgxUrlAsyncFetcher();
+
+  // It should be called in the module init_process callback function. Do some
+  // intializations which can't be done in the master process
+  bool Init(ngx_cycle_t* cycle);
+
+  // shutdown all the fetches.
+  virtual void ShutDown();
+
+  // the read handler in the main thread
+  static void ReadCallback(const ps_event_data& data);
+
+  virtual bool SupportsHttps() const { return false; }
+
+  virtual void Fetch(const GoogleString& url,
+                     MessageHandler* message_handler,
+                     AsyncFetch* callback);
+
+  bool StartFetch(NgxFetch* fetch);
+
+  // Remove the completed fetch from the active fetch set, and put it into a
+  // completed fetch list to be cleaned up.
+  void FetchComplete(NgxFetch* fetch);
+  void PrintActiveFetches(MessageHandler* handler) const;
+
+  // Indicates that it should track the original content length for
+  // fetched resources.
+  bool track_original_content_length() {
+    return track_original_content_length_;
+  }
+  void set_track_original_content_length(bool x) {
+    track_original_content_length_ = x;
+  }
+
+  typedef Pool<NgxFetch> NgxFetchPool;
+
+  // AnyPendingFetches is accurate only at the time of call; this is
+  // used conservatively during shutdown.  It counts fetches that have been
+  // requested by some thread, and can include fetches for which no action
+  // has yet been taken (ie fetches that are not active).
+  virtual bool AnyPendingFetches() {
+    return !active_fetches_.empty();
+  }
+
+  // ApproximateNumActiveFetches can under- or over-count and is used only for
+  // error reporting.
+  int ApproximateNumActiveFetches() {
+    return active_fetches_.size();
+  }
+
+  void CancelActiveFetches();
+
+  // These must be accessed with mutex_ held.
+  bool shutdown() const { return shutdown_; }
+  void set_shutdown(bool s) { shutdown_ = s; }
+
+
+ private:
+  static void TimeoutHandler(ngx_event_t* tev);
+  static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
+  friend class NgxFetch;
+
+  NgxFetchPool active_fetches_;
+  // Add the pending task to this list
+  NgxFetchPool pending_fetches_;
+  NgxFetchPool completed_fetches_;
+  ngx_url_t proxy_;
+
+  int fetchers_count_;
+  bool shutdown_;
+  bool track_original_content_length_;
+  int64 byte_count_;
+  ThreadSystem* thread_system_;
+  MessageHandler* message_handler_;
+  // Protect the member variable in this class
+  // active_fetches, pending_fetches
+  ThreadSystem::CondvarCapableMutex* mutex_;
+
+  ngx_pool_t* pool_;
+  ngx_log_t* log_;
+  ngx_resolver_t* resolver_;
+  int max_keepalive_requests_;
+  ngx_msec_t resolver_timeout_;
+  ngx_msec_t fetch_timeout_;
+
+  NgxEventConnection* event_connection_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NET_INSTAWEB_NGX_URL_ASYNC_FETCHER_H_$