Scalable ConcourseCI with Nomad and Nix

job "concourse-ci-web" {
  datacenters = ["homelab-1"]
  type = "service"

  group "svc" {
    count = 1

    network {
      mode ="bridge"

      port "db" {
        to = "5432"
      }
      port "http" {
        static = "8080"
        to = "8080"
      }
      port "tsa" {
        static = "2222"
        to = "2222"
      }
    }

    service {
      name = "concourse-web"
      port = "http"

      check {
        type = "http"
        path = "/"
        interval = "2s"
        timeout = "2s"
      }
    }

    service {
      name = "concourse-tsa"
      port = "2222"
    }

    service {
      name = "concourse-db"
      port = "db"
    }

    task "db" {
      driver = "docker"

      config {
        image = "magicrb/postgresql@sha256:changeme"
        ports = ["db"]

        volumes = [
          "secrets/main.sh:/data/scripts/main.sh",
        ]
      }

      vault {
        policies = ["concourse-db-policy"]
      }

      template {
        data = <<EOF
{{ with secret "kv/data/concourse/db" }}
USER={{ .Data.data.root_user }}
PASSWORD={{ .Data.data.root_password }}
{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/data.env"
        env = true
      }

      template {
        data = <<EOF
#!/usr/bin/env bash

env

{{ with secret "kv/data/concourse/db" }}
if process_psql -tc "SELECT 1 FROM pg_database WHERE datname = '{{ .Data.data.database }}'" | grep -q 1
then
    process_psql -c "ALTER USER {{ .Data.data.user }} WITH PASSWORD '{{ .Data.data.password }}'";
else
    process_psql -c "CREATE DATABASE {{ .Data.data.database }}"
    process_psql -c "CREATE USER {{ .Data.data.user }} WITH ENCRYPTED PASSWORD '{{ .Data.data.password }}'"
    process_psql -c "GRANT ALL PRIVILEGES ON DATABASE {{ .Data.data.database }} TO {{ .Data.data.user }}"
{{ end }}

    echo "host all all all md5" >> /data/postgresql/pg_hba.conf
    cat << EOD >> /data/postgresql/postgresql.conf
listen_addresses = '0.0.0.0'
password_encryption = md5
EOD
fi
EOF
        destination = "${NOMAD_SECRETS_DIR}/main.sh"
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "concourse/concourse@sha256:changeme"
        command = "web"
        ports = ["http", "tsa"]
      }

      vault {
        policies = ["concourse-web-policy"]
      }

      restart {
        attempts = 5
        delay = "15s"
      }

      template {
        data = <<EOF
{{ with secret "kv/data/concourse/web" }}
CONCOURSE_ADD_LOCAL_USER={{ .Data.data.local_user_name }}:{{ .Data.data.local_user_pass }}
CONCOURSE_MAIN_TEAM_LOCAL_USER={{ .Data.data.local_user_name }}
{{ end }}

CONCOURSE_SESSION_SIGNING_KEY={{ env "NOMAD_SECRETS_DIR" }}/session_signing_key
CONCOURSE_TSA_HOST_KEY={{ env "NOMAD_SECRETS_DIR" }}/tsa_host_key
CONCOURSE_TSA_AUTHORIZED_KEYS={{ env "NOMAD_SECRETS_DIR" }}/authorized_worker_keys

CONCOURSE_EXTERNAL_URL=http://blowhole.in.redalder.org:8019/

CONCOURSE_POSTGRES_HOST=127.0.0.1
CONCOURSE_POSTGRES_PORT=5432
{{ with secret "kv/data/concourse/db" }}
CONCOURSE_POSTGRES_DATABASE={{ .Data.data.database }}
CONCOURSE_POSTGRES_USER={{ .Data.data.user }}
CONCOURSE_POSTGRES_PASSWORD={{ .Data.data.password }}
{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/data.env"
        env = true
      }

      template {
        data = <<EOF
{{ with secret "kv/data/concourse/web" }}{{ .Data.data.session_signing_key }}{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/session_signing_key"
      }

      template {
        data = <<EOF
{{ with secret "kv/data/concourse/web" }}{{ .Data.data.tsa_host_key }}{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/tsa_host_key"
      }


      template {
        data = <<EOF
{{ range secrets "kv/metadata/concourse/workers/" }}
{{ with secret (printf "kv/data/concourse/workers/%s" .) }}
{{ .Data.data.public_key }}
{{ end }}
{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/authorized_worker_keys"
        change_mode = "signal"
        change_signal = "SIGHUP"
      }
    }
  }
}

concourse generate-key -t ssh -f /worker_key

_worker_key="$(cat /worker_key)"
_worker_key_pub="$(cat /worker_key.pub)"
echo -e "${_worker_key//$'\n'/\\\\n}" > /worker_key
echo -e "${_worker_key_pub//$'\n'/\\\\n}" > /worker_key.pub


JSON_FMT='{"public_key":"%s","private_key":"%s"}'
printf "$JSON_FMT" "$(< /worker_key.pub)" "$(< /worker_key)" > secret.json

vault kv put kv/concourse/workers/blowhole @secret.json

job "concourse-ci-worker" {
  datacenters = ["homelab-1"]
  type = "system"

  group "svc" {
    count = 1

    network {
      mode = "bridge"
    }

    task "create-secret" {
      driver = "docker"

      config {
        image = "useyourown"
      }

      vault {
        policies = ["concourse-worker-policy"]
      }

      lifecycle {
        sidecar = false
        hook = "prestart"
      }

      template {
        data = <<EOF
HOST_HOSTNAME="{{ env "node.unique.name" }}"
VAULT_ADDR="https://example.com:8200/"
EOF
        env = true
        destination = "${NOMAD_TASK_DIR}/data.env"
      }
    }

    task "worker" {
      driver = "docker"

      config {
        image = "concourse/concourse@sha256:changeme"
        command = "worker"
        privileged = true
      }

      vault {
        policies = ["concourse-worker-policy"]
      }

      template {
        data = <<EOF
CONCOURSE_WORK_DIR=/opt/concourse/worker
CONCOURSE_TSA_HOST=example.com:2222
CONCOURSE_TSA_PUBLIC_KEY={{ env "NOMAD_SECRETS_DIR" }}/tsa_host_key.pub
CONCOURSE_TSA_WORKER_PRIVATE_KEY={{ env "NOMAD_SECRETS_DIR" }}/worker.key
EOF
        env = true
        destination = "${NOMAD_SECRETS_DIR}/data.env"
      }

      template {
        data = <<EOF
{{ with secret (printf "kv/data/concourse/workers/%s" (env "node.unique.name") ) }}
{{ .Data.data.private_key }}
{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/worker.key"
      }

      template {
        data = <<EOF
{{ with secret "kv/data/concourse/web" }}{{ .Data.data.tsa_host_key_pub }}{{ end }}
EOF
        destination = "${NOMAD_SECRETS_DIR}/tsa_host_key.pub"
      }
    }
  }
}