BTHLABS-63: Production deployment workflow

.gitea/actions/setup-ansible/action.yaml

@@ -0,0 +1,19 @@
+name: "Set up Ansible"
+description: "Downloads and installs Ansible"
+inputs:
+  version:
+    description: "Ansible version to install"
+    required: false
+    default: "10.2.0"
+runs:
+  using: "composite"
+  steps:
+    - name: "Install Ansible"
+      shell: "bash"
+      env:
+        PIP_INDEX_URL: "https://nexus.bthlabs.pl/repository/pypi/simple/"
+      run: |
+        set -x
+        python3 -m venv /opt/ansible
+
+        /opt/ansible/bin/pip install ansible==${{ inputs.version }}

.gitea/workflows/build-deployment-images.yaml

@@ -0,0 +1,81 @@
+name: "Build deployment images"
+
+on:
+  workflow_call:
+    inputs:
+      target:
+        required: true
+        type: "string"
+      registry:
+        required: false
+        type: "string"
+        default: "docker-hosted.nexus.bthlabs.pl"
+      platform:
+        required: false
+        type: "string"
+        default: "linux/amd64,linux/arm64"
+    secrets:
+      VAULT_ROLE_ID:
+        required: true
+      VAULT_SECRET_ID:
+        required: true
+
+jobs:
+  build-deployment-images:
+    name: "Build deployment images"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "Checkout the code"
+        uses: "actions/checkout@v2"
+      - name: "Get build options"
+        id: "get-build-options"
+        uses: "./.gitea/actions/get-build-options"
+      - name: "Get `backend` version"
+        id: "get-backend-version"
+        uses: "./.gitea/actions/get-service-version"
+        with:
+          service: "backend"
+      - name: "Import Secrets"
+        id: "import-secrets"
+        uses: "hashicorp/vault-action@v2"
+        with:
+          url: "https://vault.bthlabs.pl/"
+          method: "approle"
+          roleId: "${{ secrets.VAULT_ROLE_ID }}"
+          secretId: "${{ secrets.VAULT_SECRET_ID }}"
+          secrets: |
+            gitea/data/${{ inputs.registry }} username | DOCKER_USERNAME ;
+            gitea/data/${{ inputs.registry }} password | DOCKER_PASSWORD
+      - name: "Set up Docker Buildx"
+        id: "setup-docker-buildx"
+        uses: "docker/setup-buildx-action@v3"
+        with:
+          driver: "remote"
+          endpoint: "tcp://builder-01.bthlab:2375"
+          platforms: "linux/amd64"
+          append: |
+            - endpoint: "tcp://builder-mac-01.bthlab:2375"
+              platforms: "linux/arm64"
+      - name: "Login to Docker Registry"
+        uses: "docker/login-action@v3"
+        with:
+          registry: "${{ inputs.registry }}"
+          username: "${{ steps.import-secrets.outputs.DOCKER_USERNAME }}"
+          password: "${{ steps.import-secrets.outputs.DOCKER_PASSWORD }}"
+      - name: "Build `backend-aio` image"
+        env:
+          SHORT_SHA: "${{ steps.get-build-options.outputs.short-sha }}"
+          VERSION: "${{ steps.get-backend-version.outputs.version }}"
+          BUILD: "${{ steps.get-backend-version.outputs.build-number }}"
+        run: |
+          set -x
+          docker buildx build \
+            --cache-from "type=registry,ref=nexus.bthlab.bthlabs.net:8001/hotpocket" \
+            --cache-to "type=registry,ref=nexus.bthlab.bthlabs.net:8001/hotpocket,target=max" \
+            --push \
+            --platform "${{ inputs.platform }}" \
+            --build-arg IMAGE_ID="${{ inputs.target }}.${SHORT_SHA}" \
+            -f services/backend/Dockerfile \
+            --target "${{ inputs.target }}" \
+            -t "${{ inputs.registry }}/hotpocket/backend:${{ inputs.target }}-${VERSION}-${BUILD}" \
+            services/

.gitea/workflows/development.yaml

@@ -1,4 +1,4 @@
-name: "Deploy to development"
+name: "Development deployment"
 
 on:
   push:
@@ -6,81 +6,28 @@ on:
       - "development"
 
 jobs:
-  build-deployment-images:
-    name: "Build deployment images"
-    runs-on: "ubuntu-latest"
-    steps:
-      - name: "Checkout the code"
-        uses: "actions/checkout@v2"
-      - name: "Get build options"
-        id: "get-build-options"
-        uses: "./.gitea/actions/get-build-options"
-      - name: "Get `backend` version"
-        id: "get-backend-version"
-        uses: "./.gitea/actions/get-service-version"
-        with:
-          service: "backend"
-      - name: "Import Secrets"
-        id: "import-secrets"
-        uses: "hashicorp/vault-action@v2"
-        with:
-          url: "https://vault.bthlabs.pl/"
-          method: "approle"
-          roleId: "${{ secrets.VAULT_ROLE_ID }}"
-          secretId: "${{ secrets.VAULT_SECRET_ID }}"
-          secrets: |
-            gitea/data/docker-hosted.nexus.bthlab.bthlabs.net username | DOCKER_USERNAME ;
-            gitea/data/docker-hosted.nexus.bthlab.bthlabs.net password | DOCKER_PASSWORD
-      - name: "Set up Docker Buildx"
-        id: "setup-docker-buildx"
-        uses: "docker/setup-buildx-action@v3"
-        with:
-          driver: "remote"
-          endpoint: "tcp://builder-01.bthlab:2375"
-          platforms: "linux/amd64"
-          append: |
-            - endpoint: "tcp://builder-mac-01.bthlab:2375"
-              platforms: "linux/arm64"
-      - name: "Login to Docket Registry"
-        uses: "docker/login-action@v3"
-        with:
-          registry: "nexus.bthlab.bthlabs.net:8002"
-          username: "${{ steps.import-secrets.outputs.DOCKER_USERNAME }}"
-          password: "${{ steps.import-secrets.outputs.DOCKER_PASSWORD }}"
-      - name: "Build `backend-deployment` image"
-        env:
-          SHORT_SHA: "${{ steps.get-build-options.outputs.short-sha }}"
-          VERSION: "${{ steps.get-backend-version.outputs.version }}"
-          BUILD: "${{ steps.get-backend-version.outputs.build-number }}"
-        run: |
-          set -x
-          docker buildx build \
-            --cache-from "type=registry,ref=nexus.bthlab.bthlabs.net:8001/hotpocket" \
-            --cache-to "type=registry,ref=nexus.bthlab.bthlabs.net:8001/hotpocket,mode=max" \
-            --push \
-            --platform linux/amd64 \
-            --build-arg IMAGE_ID="deployment.${SHORT_SHA}" \
-            -f services/backend/Dockerfile \
-            --target deployment \
-            -t "nexus.bthlab.bthlabs.net:8002/hotpocket/backend:deployment-${VERSION}-${BUILD}" \
-            services/
+  build-for-development:
+    name: "Build"
+    uses: "./.gitea/workflows/build-deployment-images.yaml"
+    with:
+      target: "deployment"
+      platform: "linux/amd64"
+      registry: "nexus.bthlab.bthlabs.net:8002"
+    secrets:
+      VAULT_ROLE_ID: "${{ secrets.VAULT_ROLE_ID }}"
+      VAULT_SECRET_ID: "${{ secrets.VAULT_SECRET_ID }}"
 
-  deploy:
+  deploy-to-deployment:
     name: "Deploy"
     runs-on: "ubuntu-latest"
     needs:
-      - "build-deployment-images"
+      - "build-for-development"
     env:
       KUBERNETES_NAMESPACE: "hotpocket-development"
       KUBERNETES_CLUSTER: "k8s.bthlab"
     steps:
       - name: "Checkout the code"
         uses: "actions/checkout@v2"
-      - name: "Get run info"
-        id: "get-run-info"
-        uses: "./.gitea/actions/get-run-info"
-        with:
-          compose-project-base: "${{ vars.COMPOSE_PROJECT_BASE }}"
       - name: "Get build options"
         id: "get-build-options"
         uses: "./.gitea/actions/get-build-options"
@@ -105,7 +52,6 @@ jobs:
             gitea/data/k8s.bthlab config | KUBECONFIG_PAYLOAD
       - name: "Set up kubeconfig"
         env:
-          COMPOSE_PROJECT: "${{ steps.get-run-info.outputs.compose-project }}"
           KUBECONFIG_PAYLOAD: "${{ steps.import-secrets.outputs.KUBECONFIG_PAYLOAD }}"
         run: |
           set -x
@@ -117,7 +63,6 @@ jobs:
           /opt/k8s/bin/kubectl get node
       - name: "Run `backend` Django migrations"
         env:
-          COMPOSE_PROJECT: "${{ steps.get-run-info.outputs.compose-project }}"
           BACKEND_TAG: "deployment-${{ steps.get-backend-version.outputs.version }}-${{ steps.get-backend-version.outputs.build-number }}"
         run: |
           set -x
@@ -126,6 +71,7 @@ jobs:
             cd deployment/hotpocket.bthlab ;
             export KUBECONFIG="/opt/k8s/etc/kubeconfig" ;
             /opt/k8s/bin/kubectl config use-context ${KUBERNETES_CLUSTER} ;
+            /opt/k8s/bin/kubectl -n ${KUBERNETES_NAMESPACE} apply -f resources/backend/config-map-local-deps.yaml ;
             /opt/k8s/bin/kubectl -n ${KUBERNETES_NAMESPACE} set image cronjobs/backend-job-migrations migrations=nexus.bthlab.bthlabs.net:8002/hotpocket/backend:${BACKEND_TAG} ;
             /opt/k8s/bin/kubectl -n ${KUBERNETES_NAMESPACE} delete jobs --ignore-not-found=true backend-job-migrations ;
             /opt/k8s/bin/kubectl -n ${KUBERNETES_NAMESPACE} create job backend-job-migrations --from=cronjob/backend-job-migrations ;
@@ -133,7 +79,6 @@ jobs:
           )
       - name: "Deploy"
         env:
-          COMPOSE_PROJECT: "${{ steps.get-run-info.outputs.compose-project }}"
           BACKEND_TAG: "deployment-${{ steps.get-backend-version.outputs.version }}-${{ steps.get-backend-version.outputs.build-number }}"
         run: |
           set -x

.gitea/workflows/production.yaml

@@ -0,0 +1,76 @@
+name: "Production deployment"
+
+on:
+  release:
+    types: ["published"]
+
+jobs:
+  build-for-production:
+    name: "Build"
+    uses: "./.gitea/workflows/build-deployment-images.yaml"
+    with:
+      target: "deployment"
+      platform: "linux/amd64"
+    secrets:
+      VAULT_ROLE_ID: "${{ secrets.VAULT_ROLE_ID }}"
+      VAULT_SECRET_ID: "${{ secrets.VAULT_SECRET_ID }}"
+
+  deploy-to-production:
+    name: "Deploy"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-for-production"
+    steps:
+      - name: "Checkout the code"
+        uses: "actions/checkout@v2"
+      - name: "Get build options"
+        id: "get-build-options"
+        uses: "./.gitea/actions/get-build-options"
+      - name: "Get `backend` version"
+        id: "get-backend-version"
+        uses: "./.gitea/actions/get-service-version"
+        with:
+          service: "backend"
+      - name: "Import Secrets"
+        id: "import-secrets"
+        uses: "hashicorp/vault-action@v2"
+        with:
+          url: "https://vault.bthlabs.pl/"
+          method: "approle"
+          roleId: "${{ secrets.VAULT_ROLE_ID }}"
+          secretId: "${{ secrets.VAULT_SECRET_ID }}"
+          secrets: |
+            gitea/data/hotpocket.app ansible_vault_payload | ANSIBLE_VAULT_PAYLOAD ;
+            gitea/data/hotpocket.app ansible_vault_password | ANSIBLE_VAULT_PASSWORD ;
+            gitea/data/hotpocket.app ansible_inventory_payload | ANSIBLE_INVENTORY_PAYLOAD ;
+            gitea/data/hotpocket.app ssh_key_payload | SSH_KEY_PAYLOAD
+      - name: "Setup Ansible"
+        uses: "./.gitea/actions/setup-ansible"
+      - name: "Prepare Ansible secrets"
+        run: |
+          set -x
+
+          mkdir deployment/hotpocket_app/.ci
+          echo "${ANSIBLE_VAULT_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/env_vars/production/vault.yaml"
+          echo "${ANSIBLE_VAULT_PASSWORD}" >"deployment/hotpocket_app/.ci/vault_password"
+          echo "${ANSIBLE_INVENTORY_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/inventory_ci.yaml"
+          echo "${SSH_KEY_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/.ci/ssh_key"
+          chmod 600 deployment/hotpocket_app/.ci/ssh_key
+      - name: "Engage!"
+        env:
+          VERSION: "${{ steps.get-backend-version.outputs.version }}"
+          BUILD: "${{ steps.get-backend-version.outputs.build-number }}"
+        run: |
+          set -x
+
+          (
+            cd deployment/hotpocket_app ;
+            ANSIBLE_HOST_KEY_CHECKING="False" /opt/ansible/bin/ansible-playbook \
+              -i inventory_ci.yaml \
+              --vault-id hotpocket@.ci/vault_password \
+              -e @env_vars/production/vars.yaml \
+              -e @env_vars/production/vault.yaml \
+              -e hotpocket_app_image_tag="deployment-${VERSION}-${BUILD}" \
+              --limit "*.production.hotpocket.app" \
+              deploy.yaml
+          )

.gitea/workflows/staging.yaml

@@ -0,0 +1,76 @@
+name: "Staging deployment"
+
+on:
+  release:
+    types: ["published"]
+
+jobs:
+  build-for-staging:
+    name: "Build"
+    uses: "./.gitea/workflows/build-deployment-images.yaml"
+    with:
+      target: "aio"
+      platform: "linux/amd64"
+    secrets:
+      VAULT_ROLE_ID: "${{ secrets.VAULT_ROLE_ID }}"
+      VAULT_SECRET_ID: "${{ secrets.VAULT_SECRET_ID }}"
+
+  deploy-to-staging:
+    name: "Deploy"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "build-for-staging"
+    steps:
+      - name: "Checkout the code"
+        uses: "actions/checkout@v2"
+      - name: "Get build options"
+        id: "get-build-options"
+        uses: "./.gitea/actions/get-build-options"
+      - name: "Get `backend` version"
+        id: "get-backend-version"
+        uses: "./.gitea/actions/get-service-version"
+        with:
+          service: "backend"
+      - name: "Import Secrets"
+        id: "import-secrets"
+        uses: "hashicorp/vault-action@v2"
+        with:
+          url: "https://vault.bthlabs.pl/"
+          method: "approle"
+          roleId: "${{ secrets.VAULT_ROLE_ID }}"
+          secretId: "${{ secrets.VAULT_SECRET_ID }}"
+          secrets: |
+            gitea/data/staging.hotpocket.app ansible_vault_payload | ANSIBLE_VAULT_PAYLOAD ;
+            gitea/data/staging.hotpocket.app ansible_vault_password | ANSIBLE_VAULT_PASSWORD ;
+            gitea/data/staging.hotpocket.app ansible_inventory_payload | ANSIBLE_INVENTORY_PAYLOAD ;
+            gitea/data/staging.hotpocket.app ssh_key_payload | SSH_KEY_PAYLOAD
+      - name: "Setup Ansible"
+        uses: "./.gitea/actions/setup-ansible"
+      - name: "Prepare Ansible secrets"
+        run: |
+          set -x
+
+          mkdir deployment/hotpocket_app/.ci
+          echo "${ANSIBLE_VAULT_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/env_vars/staging/vault.yaml"
+          echo "${ANSIBLE_VAULT_PASSWORD}" >"deployment/hotpocket_app/.ci/vault_password"
+          echo "${ANSIBLE_INVENTORY_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/inventory_ci.yaml"
+          echo "${SSH_KEY_PAYLOAD}" | base64 -d >"deployment/hotpocket_app/.ci/ssh_key"
+          chmod 600 deployment/hotpocket_app/.ci/ssh_key
+      - name: "Engage!"
+        env:
+          VERSION: "${{ steps.get-backend-version.outputs.version }}"
+          BUILD: "${{ steps.get-backend-version.outputs.build-number }}"
+        run: |
+          set -x
+
+          (
+            cd deployment/hotpocket_app ;
+            ANSIBLE_HOST_KEY_CHECKING="False" /opt/ansible/bin/ansible-playbook \
+              -i inventory_ci.yaml \
+              --vault-id hotpocket@.ci/vault_password \
+              -e @env_vars/staging/vars.yaml \
+              -e @env_vars/staging/vault.yaml \
+              -e hotpocket_app_image_tag="aio-${VERSION}-${BUILD}" \
+              --limit "*.staging.hotpocket.app" \
+              deploy.yaml
+          )