Utilizing GitLab Caching To Speed Up Build Process For Docker Based Project

In this post, we’ll see how we can utilize caching in a GitLab pipeline to speed up build process for a docker based project

Step 1: Create a GitLab project named kaniko-caching with following files:

Dockerfile

# Dockerfile
FROM node:16-alpine

WORKDIR /usr/src/app

COPY app/package*.json ./
RUN npm install

COPY app .

EXPOSE 3000
CMD ["node", "index.js"]

.gitlab-ci.yml

stages:
  - build_tar
  - run_container

build_tar:
  stage: build_tar
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  variables:
    DOCKER_REGISTRY: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com
    ECR_REGISTRY_NAME: scanning
  script:
    - echo "{\"credsStore\":\"ecr-login\",\"credHelpers\":{\"$DOCKER_REGISTRY\":\"ecr-login\"}}" > /kaniko/.docker/config.json
    - /kaniko/executor --context "./" --dockerfile "./Dockerfile" --no-push --destination output:latest --tarPath api.tar --cache=true --cache-repo=$DOCKER_REGISTRY/$ECR_REGISTRY_NAME --cache-run-layers --snapshot-mode=redo --cleanup --cache-ttl 720h0m0s --use-new-run
  only:
    - main
  artifacts:
    paths:
      - "api.tar"
    expire_in: 1 day

run_container:
  stage: run_container
  image: docker:latest
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
  needs:
    - build_tar
  dependencies:
    - build_tar
  script:
    - docker load -i api.tar 
    - docker images 
    - docker run --rm output:latest

app/index.js

const express = require('express');
const app = express();
const port = 3000;

app.get('/', (req, res) => {
  res.send('Hello, World!');
});

app.listen(port, () => {
  console.log(`App1 listening at http://localhost:${port}`);
});

app/package.json

{
  "name": "sample-node-app",
  "version": "1.0.0",
  "description": "A sample Node.js application",
  "main": "index.js",
  "scripts": {
    "start": "node index.js"
  },
  "author": "Your Name",
  "license": "MIT",
  "dependencies": {
    "express": "^4.17.1",
    "lodash": "^4.17.21",
    "mongoose": "^6.0.14",
    "axios": "^0.24.0",
    "cors": "^2.8.5",
    "dotenv": "^10.0.0",
    "morgan": "^1.10.0",
    "body-parser": "^1.19.0",
    "moment": "^2.29.1",
    "uuid": "^8.3.2"
  }
}

Step 2: Create following GitLab CI/CD variables

AWS_ACCESS_KEY_ID
AWS_ACCOUNT_ID
AWS_SECRET_ACCESS_KEY

Step 3: Now run the pipeline. It took 36 seconds for image build step and kaniko pushed cache layer to ECR

Now make some changes to index.js file and re-run the pipeline. This time cache was used and imge build time got reduced to 24 seconds.

Edit: Instead of using access key and secret access key, you can also create OIDC IAM role for GitLab and use it for AWS authentication. For that pipeline can be changed to this

stages:
  - get_ecr_token
  - image_build
  - run_container

.setup-script:
  before_script:
    - mkdir -p ~/.aws
    - echo "${MY_OIDC_TOKEN}" > /tmp/web_identity_token
    - echo -e
      "[default]\nrole_arn=${ROLE_ARN}\nweb_identity_token_file=/tmp/web_identity_token"
      > ~/.aws/config
  
get_ecr_token:
  stage: get_ecr_token
  variables:
    ROLE_ARN: $GITLAB_ROLE_ARN
    AWS_REGION: "us-east-1"
  image:
    name: amazon/aws-cli:latest
    entrypoint:
      - ""
  id_tokens:
    MY_OIDC_TOKEN:
      aud: https://gitlab.com
  before_script:
    !reference [.setup-script, before_script]
  script:
    - token=$(aws ecr get-login-password --region $AWS_REGION)
    - echo NONPROD_DOCKER_PASSWORD=$token > token.env
    - ECR_TOKEN=$(aws ecr get-login-password --region $AWS_REGION)
    - AUTH_TOKEN=$(echo -n "AWS:$ECR_TOKEN" | base64 -w 0)
    - echo AUTH_TOKEN=$AUTH_TOKEN > token.env
  artifacts:
    reports:
      dotenv: token.env
    
image_build:
  stage: image_build
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  variables:
    DOCKER_REGISTRY: ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com
  needs:
    - get_ecr_token 
  dependencies:
    - get_ecr_token
  script:
    - mkdir -p /kaniko/.docker
    - |
      cat <<EOF > /kaniko/.docker/config.json
        {
          "auths": {
            "${DOCKER_REGISTRY}": {
              "auth": "${AUTH_TOKEN}"
            }
          }
        }
      EOF
    - cat /kaniko/.docker/config.json
    - /kaniko/executor --context "./" --dockerfile "./Dockerfile" --no-push --destination output:latest --tarPath api.tar --cache=true --cache-repo=${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/scanning --cache-run-layers --snapshot-mode=redo --cleanup --cache-ttl 720h0m0s --use-new-run
  artifacts:
    paths:
      - "api.tar"
    expire_in: 1 day

run_container:
  stage: run_container
  image: docker:latest
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
  needs:
    - image_build
  dependencies:
    - image_build
  script:
    - docker load -i api.tar 
    - docker images 
    - docker run --rm output:latest