Utilizing GitLab Caching To Speed Up Build Process

Reference: https://blog.theodo.ma/let-s-make-faster-gitlab-ci-cd-pipelines/

In this post, we’ll see how we can utilize caching in a GitLab pipeline to cache node_modules and speed up build process.

Step 1: Create a GitLab project named caching with following files:

package.json

{
  "name": "caching",
  "version": "1.0.0",
  "main": "src/index.js",
  "scripts": {
    "start": "node src/index.js",
    "test": "jest",
    "build": "echo 'Building project...'"
  },
  "dependencies": {
    "express": "^4.17.2",
    "lodash": "^4.17.21",
    "moment": "^2.29.1",
    "axios": "^0.21.1",
    "chalk": "^4.1.0",
    "dotenv": "^10.0.0",
    "mongoose": "^5.12.3",
    "body-parser": "^1.19.0",
    "cors": "^2.8.5",
    "morgan": "^1.10.0"
  },
  "devDependencies": {
    "jest": "^26.6.3",
    "supertest": "^6.1.3"
  }
}

.gitlab-ci.yml

image: node:16

stages:
  - build
  - test
  - deploy

install_dependencies:
  stage: build
  script:
    - |
      yarn_output=$(yarn install --frozen-lockfile)
      echo "$yarn_output"
      if echo "$yarn_output" | grep -q "Already up-to-date"; then
        touch .yarn.cached
      fi
    - ls -al
    - if [ -f .yarn.cached ]; then rm -rf ./node_modules;fi  
    - ls -al
  cache:
    key:
      files:
        - yarn.lock
    paths:
      - ./node_modules
 
build_project:
  stage: build
  script:
    - echo "Running build script..."
    - yarn build
    - ls
    - ls node_modules
    - find . -path "*/node_modules/*" -delete
    - ls node_modules
  cache:
    key:
      files:
        - yarn.lock
    paths:
      - ./node_modules
    policy: pull
  needs:
    - install_dependencies
  artifacts:
    paths:
      - "**"
    expire_in: 1 month

run_tests:
  stage: test
  script:
    - ls node_modules
    - yarn test
  dependencies: []
  cache:
    key:
      files:
        - yarn.lock
    paths:
      - ./node_modules
    policy: pull
  
deploy_project:
  stage: deploy
  script:
    - echo "Deploying project..."
    - ls node_modules
  dependencies:
    - build_project
  cache:
    key:
      files:
        - yarn.lock
    paths:
      - ./node_modules
    policy: pull

src/index.js

const express = require('express');
const app = express();
const port = 3000;

app.get('/', (req, res) => {
  res.send('Hello, GitLab CI/CD!');
});

app.listen(port, () => {
  console.log(`App listening at http://localhost:${port}`);
});

src/index.ts.js

const request = require('supertest');
const express = require('express');

const app = express();

app.get('/', (req, res) => {
  res.status(200).send('Hello, GitLab CI/CD!');
});

describe('GET /', () => {
  it('responds with Hello, GitLab CI/CD!', async () => {
    const response = await request(app).get('/');
    expect(response.statusCode).toBe(200);
    expect(response.text).toBe('Hello, GitLab CI/CD!');
  });
});

Here we are caching node_modules folders based on the changes made yarn.lock file

Step 2: Now run the pipeline and check the logs of install_dependencies step. You’ll see cache getting created and uploaded.

Re-run the pipeline and check the logs of install_dependencies step. This time it will download the existing cache and use it. That’s why when yarn install command was executed, we got Already up-to-date message.