diff --git a/src/Dockerfile b/src/Dockerfile
index d9183b1..3482252 100644
--- a/src/Dockerfile
+++ b/src/Dockerfile
@@ -35,7 +35,10 @@ RUN serversideup-dep-install-debian ${PACKAGE_DEPENDENCIES} && \
     chmod 700 /home/$SSH_USER/.ssh && \
     \
     # Create run directory
-    mkdir -p /run/sshd
+    mkdir -p /run/sshd && \
+    \
+    # Remove default SSH host keys
+    rm -f /etc/ssh/ssh_host_*_key*
 
 # Expose the SSH port
 EXPOSE 2222