diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e27fa3..72ec24f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,7 @@
 
 ## Changes since v3.2.0
 
+- [#209](https://github.com/pusher/outh2_proxy/pull/209) Improve docker build caching of layers (@dekimsey)
 - [#186](https://github.com/pusher/oauth2_proxy/pull/186) Make config consistent (@JoelSpeed)
 - [#187](https://github.com/pusher/oauth2_proxy/pull/187) Move root packages to pkg folder (@JoelSpeed)
 - [#65](https://github.com/pusher/oauth2_proxy/pull/65) Improvements to authenticate requests with a JWT bearer token in the `Authorization` header via
diff --git a/Dockerfile b/Dockerfile
index 290d0f9..cc3a745 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,11 +5,14 @@ RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.s
 
 # Copy sources
 WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
-COPY . .
 
 # Fetch dependencies
+COPY go.mod go.sum ./
 RUN go mod download
 
+# Now pull in our code
+COPY . .
+
 # Build binary and make sure there is at least an empty key file.
 #  This is useful for GCP App Engine custom runtime builds, because
 #  you cannot use multiline variables in their app.yaml, so you have to
diff --git a/Dockerfile.arm64 b/Dockerfile.arm64
index efb585d..7138da8 100644
--- a/Dockerfile.arm64
+++ b/Dockerfile.arm64
@@ -5,11 +5,14 @@ RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.s
 
 # Copy sources
 WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
-COPY . .
 
 # Fetch dependencies
+COPY go.mod go.sum ./
 RUN go mod download
 
+# Now pull in our code
+COPY . .
+
 # Build binary and make sure there is at least an empty key file.
 #  This is useful for GCP App Engine custom runtime builds, because
 #  you cannot use multiline variables in their app.yaml, so you have to
diff --git a/Dockerfile.armv6 b/Dockerfile.armv6
index a741606..15db884 100644
--- a/Dockerfile.armv6
+++ b/Dockerfile.armv6
@@ -5,11 +5,14 @@ RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.s
 
 # Copy sources
 WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
-COPY . .
 
 # Fetch dependencies
+COPY go.mod go.sum ./
 RUN go mod download
 
+# Now pull in our code
+COPY . .
+
 # Build binary and make sure there is at least an empty key file.
 #  This is useful for GCP App Engine custom runtime builds, because
 #  you cannot use multiline variables in their app.yaml, so you have to
diff --git a/configure b/configure
index 959447b..26db8fe 100755
--- a/configure
+++ b/configure
@@ -112,9 +112,7 @@ check_go_env() {
 
 cd ${0%/*}
 
-if [ ! -f .env ]; then
-  rm .env
-fi
+rm -fv .env
 
 check_for make
 check_for awk