diff --git a/CHANGELOG.md b/CHANGELOG.md
index 824e37c..fec05c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,7 @@
 - [#234](https://github.com/pusher/oauth2_proxy/pull/234) Added option `-ssl-upstream-insecure-skip-validation` to skip validation of upstream SSL certificates (@jansinger)
 - [#224](https://github.com/pusher/oauth2_proxy/pull/224) Check Google group membership using hasMember to support nested groups and external users (@jpalpant)
 - [#231](https://github.com/pusher/oauth2_proxy/pull/231) Add optional group membership and email domain checks to the GitLab provider (@Overv)
+- [#226](https://github.com/pusher/oauth2_proxy/pull/226) Made setting of proxied headers deterministic based on configuration alone (@aeijdenberg)
 - [#178](https://github.com/pusher/oauth2_proxy/pull/178) Add Silence Ping Logging and Exclude Logging Paths flags (@kskewes)
 - [#209](https://github.com/pusher/oauth2_proxy/pull/209) Improve docker build caching of layers (@dekimsey)
 - [#186](https://github.com/pusher/oauth2_proxy/pull/186) Make config consistent (@JoelSpeed)
diff --git a/oauthproxy.go b/oauthproxy.go
index f3d9fd9..2418e73 100644
--- a/oauthproxy.go
+++ b/oauthproxy.go
@@ -820,32 +820,60 @@ func (p *OAuthProxy) addHeadersForProxying(rw http.ResponseWriter, req *http.Req
 		req.Header["X-Forwarded-User"] = []string{session.User}
 		if session.Email != "" {
 			req.Header["X-Forwarded-Email"] = []string{session.Email}
+		} else {
+			req.Header.Del("X-Forwarded-Email")
 		}
 	}
+
 	if p.PassUserHeaders {
 		req.Header["X-Forwarded-User"] = []string{session.User}
 		if session.Email != "" {
 			req.Header["X-Forwarded-Email"] = []string{session.Email}
+		} else {
+			req.Header.Del("X-Forwarded-Email")
 		}
 	}
+
 	if p.SetXAuthRequest {
 		rw.Header().Set("X-Auth-Request-User", session.User)
 		if session.Email != "" {
 			rw.Header().Set("X-Auth-Request-Email", session.Email)
+		} else {
+			rw.Header().Del("X-Auth-Request-Email")
 		}
-		if p.PassAccessToken && session.AccessToken != "" {
-			rw.Header().Set("X-Auth-Request-Access-Token", session.AccessToken)
+
+		if p.PassAccessToken {
+			if session.AccessToken != "" {
+				rw.Header().Set("X-Auth-Request-Access-Token", session.AccessToken)
+			} else {
+				rw.Header().Del("X-Auth-Request-Access-Token")
+			}
 		}
 	}
-	if p.PassAccessToken && session.AccessToken != "" {
-		req.Header["X-Forwarded-Access-Token"] = []string{session.AccessToken}
+
+	if p.PassAccessToken {
+		if session.AccessToken != "" {
+			req.Header["X-Forwarded-Access-Token"] = []string{session.AccessToken}
+		} else {
+			req.Header.Del("X-Forwarded-Access-Token")
+		}
 	}
-	if p.PassAuthorization && session.IDToken != "" {
-		req.Header["Authorization"] = []string{fmt.Sprintf("Bearer %s", session.IDToken)}
+
+	if p.PassAuthorization {
+		if session.IDToken != "" {
+			req.Header["Authorization"] = []string{fmt.Sprintf("Bearer %s", session.IDToken)}
+		} else {
+			req.Header.Del("Authorization")
+		}
 	}
-	if p.SetAuthorization && session.IDToken != "" {
-		rw.Header().Set("Authorization", fmt.Sprintf("Bearer %s", session.IDToken))
+	if p.SetAuthorization {
+		if session.IDToken != "" {
+			rw.Header().Set("Authorization", fmt.Sprintf("Bearer %s", session.IDToken))
+		} else {
+			rw.Header().Del("Authorization")
+		}
 	}
+
 	if session.Email == "" {
 		rw.Header().Set("GAP-Auth", session.User)
 	} else {