diff --git a/.gitignore b/.gitignore
index a5a3c2e..f6e5055 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 oauth2_proxy
 vendor
 dist
+release
 .godeps
 *.exe
 .env
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 94e66e7..889dd78 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@
   - After a successful login, you will be redirected to your original URL rather than /
 - [#35](https://github.com/pusher/oauth2_proxy/pull/35) arm and arm64 binary releases (@kskewes)
   - Add armv6 and arm64 to Makefile `release` target
+- [#37](https://github.com/pusher/oauth2_proxy/pull/37) cross build arm and arm64 docker images (@kskewes)
 
 # v3.0.0
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 17f7620..6b8c3a5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,6 +8,8 @@ Download the dependencies using [`dep`](https://github.com/golang/dep).
 ```bash
 cd $GOPATH/src/github.com # Create this directory if it doesn't exist
 git clone git@github.com:<YOUR_FORK>/oauth2_proxy pusher/oauth2_proxy
+cd pusher/oauth2_proxy
+./configure # Setup your environment variables
 make dep
 ```
 
diff --git a/Dockerfile b/Dockerfile
index 487d29b..126ce40 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@ RUN ./configure && make build
 
 # Copy binary to alpine
 FROM alpine:3.8
-RUN apk add --no-cache ca-certificates
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=builder /go/src/github.com/pusher/oauth2_proxy/oauth2_proxy /bin/oauth2_proxy
 
 ENTRYPOINT ["/bin/oauth2_proxy"]
diff --git a/Dockerfile.arm64 b/Dockerfile.arm64
new file mode 100644
index 0000000..4a8dc7d
--- /dev/null
+++ b/Dockerfile.arm64
@@ -0,0 +1,22 @@
+FROM golang:1.11-stretch AS builder
+
+# Download tools
+RUN wget -O $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64
+RUN chmod +x $GOPATH/bin/dep
+
+# Copy sources
+WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
+COPY . .
+
+# Fetch dependencies
+RUN dep ensure --vendor-only
+
+# Build binary
+RUN ./configure && GOARCH=arm64 make build
+
+# Copy binary to alpine
+FROM arm64v8/alpine:3.8
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /go/src/github.com/pusher/oauth2_proxy/oauth2_proxy /bin/oauth2_proxy
+
+ENTRYPOINT ["/bin/oauth2_proxy"]
diff --git a/Dockerfile.armv6 b/Dockerfile.armv6
new file mode 100644
index 0000000..5f7f7af
--- /dev/null
+++ b/Dockerfile.armv6
@@ -0,0 +1,22 @@
+FROM golang:1.11-stretch AS builder
+
+# Download tools
+RUN wget -O $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64
+RUN chmod +x $GOPATH/bin/dep
+
+# Copy sources
+WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
+COPY . .
+
+# Fetch dependencies
+RUN dep ensure --vendor-only
+
+# Build binary
+RUN ./configure && GOARCH=arm GOARM=6 make build
+
+# Copy binary to alpine
+FROM arm32v6/alpine:3.8
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /go/src/github.com/pusher/oauth2_proxy/oauth2_proxy /bin/oauth2_proxy
+
+ENTRYPOINT ["/bin/oauth2_proxy"]
diff --git a/Makefile b/Makefile
index bfa4bdb..a5c4514 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,20 @@ build: clean $(BINARY)
 $(BINARY):
 	CGO_ENABLED=0 $(GO) build -a -installsuffix cgo -ldflags="-X main.VERSION=${VERSION}" -o $@ github.com/pusher/oauth2_proxy
 
+.PHONY: docker
+docker:
+	docker build -f Dockerfile -t pusher/oauth2_proxy:latest .
+
+.PHONY: docker-all
+docker-all: docker
+	docker build -f Dockerfile -t pusher/oauth2_proxy:latest-amd64 .
+	docker build -f Dockerfile -t pusher/oauth2_proxy:${VERSION} .
+	docker build -f Dockerfile -t pusher/oauth2_proxy:${VERSION}-amd64 .
+	docker build -f Dockerfile.arm64 -t pusher/oauth2_proxy:latest-arm64 .
+	docker build -f Dockerfile.arm64 -t pusher/oauth2_proxy:${VERSION}-arm64 .
+	docker build -f Dockerfile.armv6 -t pusher/oauth2_proxy:latest-armv6 .
+	docker build -f Dockerfile.armv6 -t pusher/oauth2_proxy:${VERSION}-armv6 .
+
 .PHONY: test
 test: dep lint
 	$(GO) test -v -race $(go list ./... | grep -v /vendor/)