diff --git a/.gitignore b/.gitignore
index a5a3c2e..f6e5055 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 oauth2_proxy
 vendor
 dist
+release
 .godeps
 *.exe
 .env
diff --git a/.travis.yml b/.travis.yml
index f939d99..d50b3da 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,9 @@ language: go
 go:
   - 1.10.x
   - 1.11.x
+before_install:
+  - sudo apt-get install -y qemu-user-static
+  - sudo docker run --rm --privileged multiarch/qemu-user-static:register
 install:
   # Fetch dependencies
   - wget -O dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64
diff --git a/BUILDING.md b/BUILDING.md
new file mode 100644
index 0000000..267f471
--- /dev/null
+++ b/BUILDING.md
@@ -0,0 +1,52 @@
+# Building
+
+This project is setup to build on amd64.
+
+It is also possible to cross build binaries and docker images for armv6 and arm64.
+
+## Clone repo and configure
+
+```bash
+cd $GOPATH/src/github.com # Create this directory if it doesn't exist
+git clone git@github.com:<YOUR_FORK>/oauth2_proxy pusher/oauth2_proxy
+cd pusher/oauth2_proxy
+./configure # Setup your environment variables
+make dep
+```
+
+## Building amd64
+
+Build binary:
+```bash
+make
+```
+
+Build docker image:
+```bash
+make docker
+```
+
+## Building for other architectures
+
+This requires [multiarch/qemu-user-static](https://github.com/multiarch/qemu-user-static) to be installed in your system.
+On Ubuntu:
+```bash
+sudo apt install qemu-user-static
+```
+
+Register `qemu-user-static` on your system:
+```bash
+# make qemu-register
+# or
+sudo docker run --rm --privileged multiarch/qemu-user-static:register
+```
+
+Build all binaries:
+```bash
+make release
+```
+
+Build all docker images:
+```bash
+make docker-all
+```
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5c94020..5bbb33d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@
   - After a successful login, you will be redirected to your original URL rather than /
 - [#35](https://github.com/pusher/oauth2_proxy/pull/35) arm and arm64 binary releases (@kskewes)
   - Add armv6 and arm64 to Makefile `release` target
+- [#37](https://github.com/pusher/oauth2_proxy/pull/37) cross build arm and arm64 docker images (@kskewes)
 
 # v3.0.0
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 17f7620..fe34e17 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,9 +5,12 @@ To develop on this project, please fork the repo and clone into your `$GOPATH`.
 Dependencies are **not** checked in so please download those separately.
 Download the dependencies using [`dep`](https://github.com/golang/dep).
 
+
 ```bash
 cd $GOPATH/src/github.com # Create this directory if it doesn't exist
 git clone git@github.com:<YOUR_FORK>/oauth2_proxy pusher/oauth2_proxy
+cd pusher/oauth2_proxy
+./configure # Setup your environment variables
 make dep
 ```
 
diff --git a/Dockerfile.arm64 b/Dockerfile.arm64
new file mode 100644
index 0000000..253077f
--- /dev/null
+++ b/Dockerfile.arm64
@@ -0,0 +1,24 @@
+FROM golang:1.11-stretch AS builder
+
+# Download tools
+RUN wget -O $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64
+RUN chmod +x $GOPATH/bin/dep
+
+# Copy sources
+WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
+COPY . .
+
+# Fetch dependencies
+RUN dep ensure --vendor-only
+
+# Build binary
+RUN ./configure && GOARCH=arm64 make build
+
+# Copy binary to alpine
+FROM arm64v8/alpine:3.8
+COPY dist/qemu-aarch64-static /usr/bin/
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go/src/github.com/pusher/oauth2_proxy/oauth2_proxy /bin/oauth2_proxy
+RUN rm /usr/bin/qemu-*-static -f
+
+ENTRYPOINT ["/bin/oauth2_proxy"]
diff --git a/Dockerfile.armv6 b/Dockerfile.armv6
new file mode 100644
index 0000000..1daa4c4
--- /dev/null
+++ b/Dockerfile.armv6
@@ -0,0 +1,24 @@
+FROM golang:1.11-stretch AS builder
+
+# Download tools
+RUN wget -O $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64
+RUN chmod +x $GOPATH/bin/dep
+
+# Copy sources
+WORKDIR $GOPATH/src/github.com/pusher/oauth2_proxy
+COPY . .
+
+# Fetch dependencies
+RUN dep ensure --vendor-only
+
+# Build binary
+RUN ./configure && GOARCH=arm GOARM=6 make build
+
+# Copy binary to alpine
+FROM arm32v6/alpine:3.8
+COPY dist/qemu-arm-static /usr/bin/
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go/src/github.com/pusher/oauth2_proxy/oauth2_proxy /bin/oauth2_proxy
+RUN rm /usr/bin/qemu-*-static -f
+
+ENTRYPOINT ["/bin/oauth2_proxy"]
diff --git a/Makefile b/Makefile
index bfa4bdb..8d388cf 100644
--- a/Makefile
+++ b/Makefile
@@ -45,6 +45,34 @@ build: clean $(BINARY)
 $(BINARY):
 	CGO_ENABLED=0 $(GO) build -a -installsuffix cgo -ldflags="-X main.VERSION=${VERSION}" -o $@ github.com/pusher/oauth2_proxy
 
+# Ubuntu 18.04.1 Qemu version
+QEMU-STATIC-VERSION=v2.11.1
+
+.PHONY: qemu-static
+qemu-static:
+	wget -O dist/qemu-amd64-static https://github.com/multiarch/qemu-user-static/releases/download/${QEMU-STATIC-VERSION}/qemu-x86_64-static
+	wget -O dist/qemu-aarch64-static https://github.com/multiarch/qemu-user-static/releases/download/${QEMU-STATIC-VERSION}/qemu-aarch64-static
+	wget -O dist/qemu-arm-static https://github.com/multiarch/qemu-user-static/releases/download/${QEMU-STATIC-VERSION}/qemu-arm-static
+	chmod +x dist/qemu-*-static
+
+.PHONY: qemu-register
+qemu-register:
+	docker run --rm --privileged multiarch/qemu-user-static:register
+
+.PHONY: docker
+docker:
+	docker build -f Dockerfile -t pusher/oauth2_proxy:latest .
+
+.PHONY: docker-all
+docker-all: qemu-static docker
+	docker build -f Dockerfile -t pusher/oauth2_proxy:latest-amd64 .
+	docker build -f Dockerfile -t pusher/oauth2_proxy:${VERSION} .
+	docker build -f Dockerfile -t pusher/oauth2_proxy:${VERSION}-amd64 .
+	docker build -f Dockerfile.arm64 -t pusher/oauth2_proxy:latest-arm64 .
+	docker build -f Dockerfile.arm64 -t pusher/oauth2_proxy:${VERSION}-arm64 .
+	docker build -f Dockerfile.armv6 -t pusher/oauth2_proxy:latest-armv6 .
+	docker build -f Dockerfile.armv6 -t pusher/oauth2_proxy:${VERSION}-armv6 .
+
 .PHONY: test
 test: dep lint
 	$(GO) test -v -race $(go list ./... | grep -v /vendor/)
diff --git a/README.md b/README.md
index 910a671..827b046 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,8 @@ A list of changes can be seen in the [CHANGELOG](CHANGELOG.md).
 
     c. Using the prebuilt docker image [quay.io/pusher/oauth2_proxy](https://quay.io/pusher/oauth2_proxy)
 
+    d. Building from scratch following our [Building](BUILDING.md) guide
+
 Prebuilt binaries can be validated by extracting the file and verifying it against the `sha256sum.txt` checksum file provided for each release starting with version `v3.0.0`.
 
 ```