I've seen a high number of issues on Github, however unresolved... Or after a while just closed due to lack of feedback:
#345
I'm trying to implement a varnish / turpentine configuration.
Problem:
New session user tries to add item to cart.
Adding the item to the cart however takes very long.
This is the HTTP response for a add to cart:
Request URL:https://testing.two-store.be/ajaxcart/index/add/uenc/aHR0cHM6Ly90ZXN0aW5nLnR3by1zdG9yZS5iZS9wYW50cy1hbmQtdHJvdXNlcnMtZGlhcmEuaHRtbD9fX19TSUQ9VQ,,/product/4021/form_key/4Z3KMfgvpJFprFtN/
Request Method:POST
Status Code:200
Remote Address:77.243.239.145:443
Referrer Policy:no-referrer-when-downgrade
Response Headers
cache-control:no-store, no-cache, must-revalidate, post-check=0, pre-check=0
content-encoding:gzip
content-length:13302
content-type:text/html; charset=UTF-8
date:Wed, 24 May 2017 11:42:37 GMT
expires:Thu, 19 Nov 1981 08:52:00 GMT
pragma:no-cache
server:Apache
set-cookie:frontend=bsu31mrtajqdu3lr4q9ocjdaf3; expires=Wed, 24-May-2017 12:42:36 GMT; Max-Age=3600; path=/; domain=testing.two-store.be; httponly
status:200
strict-transport-security:max-age=15768000
vary:Accept-Encoding
x-frame-options:SAMEORIGIN
x
-powered-by:PHP/5.5.38-4+deb.sury.org~xenial+1
Request Headers
:authority:testing.two-store.be
:method:POST
:path:/ajaxcart/index/add/uenc/aHR0cHM6Ly90ZXN0aW5nLnR3by1zdG9yZS5iZS9wYW50cy1hbmQtdHJvdXNlcnMtZGlhcmEuaHRtbD9fX19TSUQ9VQ,,/product/4021/form_key/4Z3KMfgvpJFprFtN/
:scheme:https
accept:application/json, text/javascript, */*; q=0.01
accept-encoding:gzip, deflate, br
accept-language:nl,en-US;q=0.8,en;q=0.6,fr;q=0.4
content-length:73
content-type:application/x-www-form-urlencoded; charset=UTF-8
cookie:email=none; external_no_cache=1; frontend=bsu31mrtajqdu3lr4q9ocjdaf3; frontend_cid=tKytwVtDVUpb26Eo
dnt:1
origin:https://testing.two-store.be
referer:https://testing.two-store.be/pants-and-trousers-diara.html
user-agent:Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36
x-requested-with:XMLHttpRequest
Form Data
view source
view URL encoded
product:4021
related_product:
super_attribute[172]:114
qty:1
isAjax:1
This is the HTTP response for a normal product page:
Request URL:https://testing.two-store.be/pants-and-trousers-diara.html
Request Method:GET
Status Code:200
Remote Address:77.243.239.145:443
Referrer Policy:no-referrer-when-downgrade
Response Headers
accept-ranges:bytes
age:15
content-encoding:gzip
content-type:text/html; charset=UTF-8
date:Wed, 24 May 2017 11:04:44 GMT
server:Apache
status:200
strict-transport-security:max-age=15768000
vary:Accept-Encoding
via:1.1 varnish-v4
x-frame-options:SAMEORIGIN
x-powered-by:PHP/5.5.38-4+deb.sury.org~xenial+1
x-turpentine-cache:1
x-turpentine-esi:1
x-varnish:360706 327856
x-varnish-currency:
x-varnish-esi-access:
x-varnish-esi-method:
x-varnish-hits:2
x-varnish-host:testing.two-store.be
x-varnish-set-cookie:frontend=pg1mrqdphdhf0kek2v8o966a84; expires=Wed, 24-May-2017 12:04:28 GMT; Max-Age=3600; path=/; domain=testing.two-store.be; httponly
x-varnish-store:
x-varnish-url:/pants-and-trousers-diara.html
Request Headers
:authority:testing.two-store.be
:method:GET
:path:/pants-and-trousers-diara.html
:scheme:https
accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
accept-encoding:gzip, deflate, sdch, br
accept-language:nl,en-US;q=0.8,en;q=0.6,fr;q=0.4
cache-control:max-age=0
cookie:frontend_cid=skjrpXgfQD1XrNjS; email=none; external_no_cache=1; frontend=pg1mrqdphdhf0kek2v8o966a84
dnt:1
referer:https://testing.two-store.be/ladies-8.html?___SID=U
upgrade-insecure-requests:1
user-agent:Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36
This is the setup:
testing.two-store.be
Magento version 1.9.1.1
Theme: BEWEAR One Theme
Configuration file:
/etc/varnish/default.vcl
backend othello {
.host = "127.0.0.1";
.port = "80";
}
backend othellotesting {
.host = "127.0.0.1";
.port = "81";
}
backend pakjes {
.host = "127.0.0.1";
.port = "82";
}
backend othellotesting2 {
.host = "127.0.0.1";
.port = "83";
}
backend twostoretesting {
.host = "127.0.0.1";
.port = "84";
}
backend lookbook {
.host = "127.0.0.1";
.port = "85";
}
backend twostore {
.host = "127.0.0.1";
.port = "86";
}
sub vcl_recv { if (std.port(server.ip) == 7080) {
set req.backend_hint = othello;
}
elsif (std.port(server.ip) == 7081) {
set req.backend_hint = othellotesting;
}
elsif (std.port(server.ip) == 7082) {
set req.backend_hint = pakjes;
}
elsif (std.port(server.ip) == 7083) {
set req.backend_hint = othellotesting2;
}
elsif (std.port(server.ip) == 7084) {
set req.backend_hint = twostoretesting;
}
elsif (std.port(server.ip) == 7085) {
set req.backend_hint = lookbook;
}
elsif (std.port(server.ip) == 7086) {
set req.backend_hint = twostore;
}
}
sub vcl_backend_response {
}
sub vcl_deliver {
}
Daemon opts:
/etc/default/varnish
START=yes
NFILES=131072
MEMLOCK=82000
DAEMON_OPTS="-a :7080,:7081,:7082,:7083,:7084,:7085,:7086 \
-T localhost:6082 \
-f /etc/varnish/default.vcl \
-S /etc/varnish/secret \
-s malloc,256m \
-p cli_buffer=16384 \
-p feature=+esi_ignore_other_elements \
-p vcc_allow_inline_c=on" \
-p http_resp_hdr_len=42000 \
-p esi_syntax=0x2 \
-p vcc_allow_inline_c=on \
Then after applying the config, this is the running config I get when I press the button:
"Download Varnish Config"
vcl 4.0;
C{
#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <pthread.h>
static pthread_mutex_t lrand_mutex = PTHREAD_MUTEX_INITIALIZER;
void generate_uuid(char* buf) {
pthread_mutex_lock(&lrand_mutex);
long a = lrand48();
long b = lrand48();
long c = lrand48();
long d = lrand48();
pthread_mutex_unlock(&lrand_mutex);
sprintf(buf, "frontend=%08lx%04lx%04lx%04lx%04lx%08lx",
a,
b & 0xffff,
(b & ((long)0x0fff0000) >> 16) | 0x4000,
(c & 0x0fff) | 0x8000,
(c & (long)0xffff0000) >> 16,
d
);
return;
}
}C
import std;
import directors;
backend othello {
.host = "127.0.0.1";
.port = "80";
}
backend othellotesting {
.host = "127.0.0.1";
.port = "81";
}
backend pakjes {
.host = "127.0.0.1";
.port = "82";
}
backend othellotesting2 {
.host = "127.0.0.1";
.port = "83";
}
backend twostoretesting {
.host = "127.0.0.1";
.port = "84";
}
backend lookbook {
.host = "127.0.0.1";
.port = "85";
}
backend twostore {
.host = "127.0.0.1";
.port = "86";
}
sub vcl_recv { if (std.port(server.ip) == 7080) {
set req.backend_hint = othello;
}
elsif (std.port(server.ip) == 7081) {
set req.backend_hint = othellotesting;
}
elsif (std.port(server.ip) == 7082) {
set req.backend_hint = pakjes;
}
elsif (std.port(server.ip) == 7083) {
set req.backend_hint = othellotesting2;
}
elsif (std.port(server.ip) == 7084) {
set req.backend_hint = twostoretesting;
}
elsif (std.port(server.ip) == 7085) {
set req.backend_hint = lookbook;
}
elsif (std.port(server.ip) == 7086) {
set req.backend_hint = twostore;
}
}
sub vcl_backend_response {
}
sub vcl_deliver {
}
backend default {
.host = "127.0.0.1";
.port = "84";
.first_byte_timeout = 300s;
.between_bytes_timeout = 300s;
}
backend admin {
.host = "127.0.0.1";
.port = "84";
.first_byte_timeout = 21600s;
.between_bytes_timeout = 21600s;
}
acl crawler_acl {
"127.0.0.1";
}
acl debug_acl {
}
sub generate_session {
if (req.url ~ ".*[&?]SID=([^&]+).*") {
set req.http.X-Varnish-Faked-Session = regsub(
req.url, ".*[&?]SID=([^&]+).*", "frontend=\1");
} else {
C{
char uuid_buf [50];
generate_uuid(uuid_buf);
static const struct gethdr_s VGC_HDR_REQ_VARNISH_FAKED_SESSION =
{ HDR_REQ, "\030X-Varnish-Faked-Session:"};
VRT_SetHdr(ctx,
&VGC_HDR_REQ_VARNISH_FAKED_SESSION,
uuid_buf,
vrt_magic_string_end
);
}C
}
if (req.http.Cookie) {
std.collect(req.http.Cookie);
set req.http.Cookie = req.http.X-Varnish-Faked-Session +
"; " + req.http.Cookie;
} else {
set req.http.Cookie = req.http.X-Varnish-Faked-Session;
}
}
sub generate_session_expires {
C{
time_t now = time(NULL);
struct tm now_tm = *gmtime(&now);
now_tm.tm_sec += 3600;
mktime(&now_tm);
char date_buf [50];
strftime(date_buf, sizeof(date_buf)-1, "%a, %d-%b-%Y %H:%M:%S %Z", &now_tm);
static const struct gethdr_s VGC_HDR_RESP_COOKIE_EXPIRES =
{ HDR_RESP, "\031X-Varnish-Cookie-Expires:"};
VRT_SetHdr(ctx,
&VGC_HDR_RESP_COOKIE_EXPIRES,
date_buf,
vrt_magic_string_end
);
}C
}
sub vcl_init {
}
sub vcl_recv {
if ( (req.http.host ~ "^(?i)www.testing.two-store.be" || req.http.host ~ "^(?i)testing.two-store.be") && req.http.X-Forwarded-Proto !~ "(?i)https") {
return (synth(750, ""));
}
if (req.restarts == 0) {
if (req.http.X-Forwarded-For) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (!true || req.http.Authorization ||
req.method !~ "^(GET|HEAD|OPTIONS)$" ||
req.http.Cookie ~ "varnish_bypass=1") {
return (pipe);
}
if(false) {
set req.http.X-Varnish-Origin-Url = req.url;
}
set req.url = regsuball(req.url, "(.*)//+(.*)", "\1/\2");
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "\*|gzip") {
set req.http.Accept-Encoding = "gzip";
} else if (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
if (req.http.User-Agent ~ "iP(?:hone|ad|od)|BlackBerry|Palm|Googlebot-Mobile|Mobile|mobile|mobi|Windows Mobile|Safari Mobile|Android|Opera (?:Mini|Mobi)") {
set req.http.X-Normalized-User-Agent = "mobile";
} else {
set req.http.X-Normalized-User-Agent = "other";
}
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
set req.http.X-Turpentine-Secret-Handshake = "1";
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?admin") {
set req.backend_hint = admin;
return (pipe);
} else {
}
if (req.http.Cookie ~ "\bcurrency=") {
set req.http.X-Varnish-Currency = regsub(
req.http.Cookie, ".*\bcurrency=([^;]*).*", "\1");
}
if (req.http.Cookie ~ "\bstore=") {
set req.http.X-Varnish-Store = regsub(
req.http.Cookie, ".*\bstore=([^;]*).*", "\1");
}
if (req.url ~ "/turpentine/esi/get(?:Block|FormKey)/") {
set req.http.X-Varnish-Esi-Method = regsub(
req.url, ".*/method/(\w+)/.*", "\1");
set req.http.X-Varnish-Esi-Access = regsub(
req.url, ".*/access/(\w+)/.*", "\1");
if (req.http.X-Varnish-Esi-Method == "esi" && req.esi_level == 0 &&
!(true || client.ip ~ debug_acl)) {
return (synth(403, "External ESI requests are not allowed"));
}
}
if (req.http.Cookie !~ "frontend=" && !req.http.X-Varnish-Esi-Method) {
if (client.ip ~ crawler_acl ||
req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*|.*Siege/.*|magespeedtest\.com|Nexcessnet_Turpentine/.*|.*PTST.*)$") {
set req.http.Cookie = "frontend=crawler-session";
} else {
call generate_session;
}
}
if (true &&
req.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
unset req.http.Cookie;
unset req.http.X-Varnish-Faked-Session;
set req.http.X-Varnish-Static = 1;
return (hash);
}
if (req.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?(?:admin|api|cron\.php|customer|account|login|register|index\.php|admin|onepage|onestepcheckout|checkout|cart|review|sales|wishlist|monkey|ebizautoresponder|oauth|tag|review|ajaxcart)" ||
req.url ~ "\?.*__from_store=") {
return (pipe);
}
if (true &&
req.url ~ "(?:[?&](?:__SID|XDEBUG_PROFILE|product|related_product|qty|isAjax)(?=[&=]|$))") {
return (pass);
}
if (req.url ~ "[?&](utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
if (true && req.url ~ "[?&](utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=") {
set req.url = regsuball(req.url, "(?:(\?)?|&)(?:utm_source|utm_medium|utm_campaign|utm_content|utm_term|gclid|cx|ie|cof|siteurl)=[^&]+", "\1");
set req.url = regsuball(req.url, "(?:(\?)&|\?$)", "\1");
}
if(false) {
set req.http.X-Varnish-Cache-Url = req.url;
set req.url = req.http.X-Varnish-Origin-Url;
unset req.http.X-Varnish-Origin-Url;
}
return (hash);
}
}
sub vcl_pipe {
unset bereq.http.X-Turpentine-Secret-Handshake;
set bereq.http.Connection = "close";
}
sub vcl_hash {
std.log("vcl_hash start");
if (true && req.http.X-Varnish-Static) {
std.log("hash_data static file - req.url: " + req.url);
hash_data(req.url);
if (req.http.Accept-Encoding) {
std.log("hash_data static file - Accept-Encoding: " + req.http.Accept-Encoding);
hash_data(req.http.Accept-Encoding);
}
std.log("vcl_hash end return lookup");
return (lookup);
}
if(false && req.http.X-Varnish-Cache-Url) {
hash_data(req.http.X-Varnish-Cache-Url);
std.log("hash_data - X-Varnish-Cache-Url: " + req.http.X-Varnish-Cache-Url);
} else {
hash_data(req.url);
std.log("hash_data - req.url: " + req.url );
}
if (req.http.Host) {
hash_data(req.http.Host);
std.log("hash_data - req.http.Host: " + req.http.Host);
} else {
hash_data(server.ip);
}
std.log("hash_data - req.http.Ssl-Offloaded: " + req.http.Ssl-Offloaded);
hash_data(req.http.Ssl-Offloaded);
if (req.http.X-Normalized-User-Agent) {
hash_data(req.http.X-Normalized-User-Agent);
std.log("hash_data - req.http.X-Normalized-User-Agent: " + req.http.X-Normalized-User-Agent);
}
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
std.log("hash_data - req.http.Accept-Encoding: " + req.http.Accept-Encoding);
}
if (req.http.X-Varnish-Store || req.http.X-Varnish-Currency) {
hash_data("s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
std.log("hash_data - Store and Currency: " + "s=" + req.http.X-Varnish-Store + "&c=" + req.http.X-Varnish-Currency);
}
if (req.http.X-Varnish-Esi-Access == "private" &&
req.http.Cookie ~ "frontend=") {
std.log("hash_data - frontned cookie: " + regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
hash_data(regsub(req.http.Cookie, "^.*?frontend=([^;]*);*.*$", "\1"));
}
if (req.http.X-Varnish-Esi-Access == "customer_group" &&
req.http.Cookie ~ "customer_group=") {
hash_data(regsub(req.http.Cookie, "^.*?customer_group=([^;]*);*.*$", "\1"));
}
std.log("vcl_hash end return lookup");
return (lookup);
}
sub vcl_hit {
}
sub vcl_backend_response {
set beresp.grace = 15s;
set beresp.http.X-Varnish-Host = bereq.http.host;
set beresp.http.X-Varnish-URL = bereq.url;
if (bereq.url ~ "^(/media/|/skin/|/js/|/)(?:(?:index|litespeed)\.php/)?") {
unset beresp.http.Vary;
set beresp.do_gzip = true;
if (beresp.status != 200 && beresp.status != 404) {
set beresp.ttl = 15s;
set beresp.uncacheable = true;
return (deliver);
} else {
if (beresp.http.Set-Cookie) {
set beresp.http.X-Varnish-Set-Cookie = beresp.http.Set-Cookie;
unset beresp.http.Set-Cookie;
}
unset beresp.http.Cache-Control;
unset beresp.http.Expires;
unset beresp.http.Pragma;
unset beresp.http.Cache;
unset beresp.http.Age;
if (beresp.http.X-Turpentine-Esi == "1") {
set beresp.do_esi = true;
}
if (beresp.http.X-Turpentine-Cache == "0") {
set beresp.ttl = 15s;
set beresp.uncacheable = true;
return (deliver);
} else {
if (true &&
bereq.url ~ ".*\.(?:css|js|jpe?g|png|gif|ico|swf)(?=\?|&|$)") {
set beresp.ttl = 28800s;
set beresp.http.Cache-Control = "max-age=28800";
} elseif (bereq.http.X-Varnish-Esi-Method) {
if (bereq.http.X-Varnish-Esi-Access == "private" &&
bereq.http.Cookie ~ "frontend=") {
set beresp.http.X-Varnish-Session = regsub(bereq.http.Cookie,
"^.*?frontend=([^;]*);*.*$", "\1");
}
if (bereq.http.X-Varnish-Esi-Method == "ajax" &&
bereq.http.X-Varnish-Esi-Access == "public") {
set beresp.http.Cache-Control = "max-age=" + regsub(
bereq.url, ".*/ttl/(\d+)/.*", "\1");
}
set beresp.ttl = std.duration(
regsub(
bereq.url, ".*/ttl/(\d+)/.*", "\1s"),
300s);
if (beresp.ttl == 0s) {
set beresp.ttl = 15s;
set beresp.uncacheable = true;
return (deliver);
}
} else {
set beresp.ttl = 3600s;
}
}
}
return (deliver);
}
}
sub vcl_synth {
if (resp.status == 750) {
set resp.status = 301;
set resp.http.Location = "https://" + req.http.host + req.url;
return(deliver);
}
}
sub vcl_deliver {
if (req.http.X-Varnish-Faked-Session) {
call generate_session_expires;
set resp.http.Set-Cookie = req.http.X-Varnish-Faked-Session +
"; expires=" + resp.http.X-Varnish-Cookie-Expires + "; path=/";
if (req.http.Host) {
if (req.http.User-Agent ~ "^(?:ApacheBench/.*|.*Googlebot.*|JoeDog/.*|.*Siege/.*|magespeedtest\.com|Nexcessnet_Turpentine/.*|.*PTST.*)$") {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
} else {
if (req.http.Host ~ "" && "" ~ "..") {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=";
} else {
set resp.http.Set-Cookie = resp.http.Set-Cookie +
"; domain=" + regsub(req.http.Host, ":\d+$", "");
}
}
}
set resp.http.Set-Cookie = resp.http.Set-Cookie + "; httponly";
unset resp.http.X-Varnish-Cookie-Expires;
}
if (req.http.X-Varnish-Esi-Method == "ajax" && req.http.X-Varnish-Esi-Access == "private") {
set resp.http.Cache-Control = "no-cache";
}
if (true || client.ip ~ debug_acl) {
set resp.http.X-Varnish-Hits = obj.hits;
set resp.http.X-Varnish-Esi-Method = req.http.X-Varnish-Esi-Method;
set resp.http.X-Varnish-Esi-Access = req.http.X-Varnish-Esi-Access;
set resp.http.X-Varnish-Currency = req.http.X-Varnish-Currency;
set resp.http.X-Varnish-Store = req.http.X-Varnish-Store;
} else {
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Turpentine-Cache;
unset resp.http.X-Turpentine-Esi;
unset resp.http.X-Turpentine-Flush-Events;
unset resp.http.X-Turpentine-Block;
unset resp.http.X-Varnish-Session;
unset resp.http.X-Varnish-Host;
unset resp.http.X-Varnish-URL;
unset resp.http.X-Varnish-Set-Cookie;
}
}
Take a look at the magento backend configuration:
Kind regards,
Pieter-Jan
MESSAGE HERE">
129 Apr 1, 2022
332 Dec 30, 2022
113 Dec 14, 2022
109 Jan 3, 2023
0 Oct 7, 2022
51 Dec 5, 2022
92 Dec 7, 2022
120 Oct 10, 2022
1 Jan 3, 2022
21 Nov 30, 2022
21 Nov 2, 2021
10 Sep 16, 2021
27 Nov 10, 2022
71 Dec 7, 2022
243 Dec 21, 2022