From adfc71ea22d8ac3ccf469dd3a1fd7877f5f35ccf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=AD=99=E6=8C=AF=E5=AE=87?= <>
Date: Tue, 4 Feb 2025 10:23:25 +0800
Subject: [PATCH] refactor(SourceFetcher, ServiceLanguage,
 DependenciesResolver): improve workspace handling, standardize language
 naming, and streamline dependency resolution
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 孙振宇 <>
---
 .../devops/builtins/commitlint/default.js     |  30 +++
 .../devops/ChangedComponentsDetector.groovy   |  31 +++
 .../devops/CommitMessageLinter.groovy         |  28 +++
 .../devops/DependenciesResolver.groovy        |  19 +-
 .../com/freeleaps/devops/SourceFetcher.groovy |   6 +-
 .../devops/enums/ServiceLanguage.groovy       |  10 +-
 first-class-pipeline/tests/Jenkinsfile        |  55 +++++-
 .../vars/executeFreeleapsPipeline.groovy      | 178 +++++++++++++-----
 8 files changed, 283 insertions(+), 74 deletions(-)
 create mode 100644 first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js
 create mode 100644 first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy
 create mode 100644 first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy

diff --git a/first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js b/first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js
new file mode 100644
index 00000000..c2ffe91e
--- /dev/null
+++ b/first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js
@@ -0,0 +1,30 @@
+module.exports = {
+  extends: ["@commitlint/config-angular"], // Extends with the Angular commitlint configuration
+  rules: {
+    "type-enum": [
+      2,
+      "always",
+      [
+        "feat",
+        "fix",
+        "docs",
+        "style",
+        "refactor",
+        "perf",
+        "test",
+        "build",
+        "ci",
+        "chore",
+        "revert", // Add or remove types as needed
+      ],
+    ],
+    "type-case": [2, "always", "lower-case"], // Type must be in lower case
+    "type-empty": [2, "never"], // Type must not be empty
+    "scope-empty": [2, "never"], // Scope must not be empty
+    "scope-case": [2, "always", "lower-case"], // Scope must be in lower case
+    "subject-empty": [2, "never"], // Subject must not be empty
+    "subject-case": [2, "never", []], // Subject must be in sentence case
+    "subject-full-stop": [2, "never", "."], // Subject must not end with a period
+    "header-max-length": [2, "always", 100], // Header must not exceed 100 characters
+  },
+};
diff --git a/first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy b/first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy
new file mode 100644
index 00000000..ac38ebcd
--- /dev/null
+++ b/first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy
@@ -0,0 +1,31 @@
+package com.freeleaps.devops
+
+class ChangedComponentsDetector {
+  def steps
+  def workspace
+
+  ChangedComponentsDetector(steps) {
+    this.steps = steps
+  }
+
+  def detect(workspace, components) {
+    def changedComponents = [] as Set
+
+    dir(workspace) {
+      // using git command to get changed files list
+      def changedFiles = sh(script: 'git diff --name-only HEAD~1 HEAD', returnStdout: true)
+        .trim()
+        .split('\n')
+      
+      changedFiles.each { file ->
+        components.each { component ->
+          if (file.startsWith("${component}/")) {
+            changedComponents.add(component)
+          }
+        }
+      }
+    }
+
+    return changedComponents.toList()
+  }
+}
\ No newline at end of file
diff --git a/first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy b/first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy
new file mode 100644
index 00000000..e33e8955
--- /dev/null
+++ b/first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy
@@ -0,0 +1,28 @@
+package com.freeleaps.devops
+
+class CommitMessageLinter {
+  def steps
+  private defaultRule = 'com/freeleaps/devops/builtins/commitlint/default.js'
+
+  CommitMessageLinter(steps) {
+    this.steps = steps
+  }
+
+  def lint(configurations) {
+    def rules = steps.libraryResource 'com/freeleaps/devops/builtins/commitlint/default.js'
+    steps.log.info "Check if there has custom commit lint rules specified..."
+
+    if (configurations.commitLintRules != null && !configurations.commitLintRules.isEmpty()) {
+      steps.log.info "Custom commit lint rules found, using custom rules files: ${configurations.commitLintRules}"
+      rules = configurations.commitLintRules
+    } else {
+      steps.log.info "No custom commit lint rules found, using built-in rules at: ${defaultRules}"
+      steps.sh "echo ${rules} > .commitlintrc.js"
+      rules = '.commitlintrc.js'
+    }
+
+    steps.log.info "Linting commit messages from HEAD..."
+
+    steps.sh "commitlint --verbose -g ${rules} -f HEAD^"
+  }
+}
\ No newline at end of file
diff --git a/first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy b/first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy
index 2002533a..a074b207 100644
--- a/first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy
+++ b/first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy
@@ -43,19 +43,8 @@ class DependenciesResolver {
 
     switch (mgr) {
       case DependenciesManager.PIP:
-        if (configurations.pipRequirementsFile == null || configurations.pipRequirementsFile.isEmpty()) {
-          steps.error("pipRequirementsFile is required when using PIP as dependencies manager")
-        }
-
-        def requirementsFile = configurations.pipRequirementsFile
-
-        if (cachingEnabled) {
-          steps.cache(maxCacheSize: 512, caches: [[$class: 'ArbitraryFileCache', excludes: '', includes: '**/*', path: '.pip-cache']]) {
-            steps.sh "pip install -r ${requirementsFile} --cache-dir .pip-cache"
-          }
-        } else {
-          steps.sh "pip install -r ${requirementsFile}"
-        }
+        steps.log.warn "Python project no need to resolving dependencies, skipping..."
+        break
       case DependenciesManager.NPM:
         if (configurations.npmPackageJsonFile == null || configurations.npmPackageJsonFile.isEmpty()) {
           steps.error("npmPackageJsonFile is required when using NPM as dependencies manager")
@@ -70,6 +59,8 @@ class DependenciesResolver {
         } else {
           steps.sh "npm install"
         }
+
+        break
       case DependenciesManager.YARN:
         if (configurations.yarnPackageJsonFile == null || configurations.yarnPackageJsonFile.isEmpty()) {
           steps.error("yarnPackageJsonFile is required when using YARN as dependencies manager")
@@ -84,6 +75,8 @@ class DependenciesResolver {
         } else {
           steps.sh "yarn install"
         }
+
+        break
       default:
         steps.error("Unsupported dependencies manager")
     }
diff --git a/first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy b/first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy
index f9c865a0..1182b161 100644
--- a/first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy
+++ b/first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy
@@ -16,6 +16,10 @@ class SourceFetcher {
       steps.error("serviceGitBranch is required")
     }
 
-    steps.git branch: configurations.serviceGitBranch, credentialsId: 'git-bot-credentials', url: configurations.serviceGitRepo
+    steps.env.workspace = "workspace"
+
+    dir(steps.env.workspace) {
+      steps.git branch: configurations.serviceGitBranch, credentialsId: 'git-bot-credentials', url: configurations.serviceGitRepo
+    }
   }
 }
\ No newline at end of file
diff --git a/first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy b/first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy
index 392314d7..261ec770 100644
--- a/first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy
+++ b/first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy
@@ -2,8 +2,8 @@ package com.freeleaps.devops.enums
 
 enum ServiceLanguage {
 
-  PYTHON('Python'),
-  NODE('Node (JS,TS)'),
+  PYTHON('python'),
+  JS('javascript'),
   UNKNOWN('Unknown')
 
   final String language
@@ -14,10 +14,10 @@ enum ServiceLanguage {
 
   static ServiceLanguage parse(String language) {
     switch (language) {
-      case 'Python':
+      case 'python':
         return ServiceLanguage.PYTHON
-      case 'Node (JS,TS)':
-        return ServiceLanguage.NODE
+      case 'javascript':
+        return ServiceLanguage.JS
       default:
         return ServiceLanguage.UNKNOWN
     }
diff --git a/first-class-pipeline/tests/Jenkinsfile b/first-class-pipeline/tests/Jenkinsfile
index f9a68fc7..a35ff642 100644
--- a/first-class-pipeline/tests/Jenkinsfile
+++ b/first-class-pipeline/tests/Jenkinsfile
@@ -2,11 +2,56 @@ library 'first-class-pipeline'
 
 executeFreeleapsPipeline {
   serviceName = 'magicleaps'
-  serviceLang = 'Python'
+  environmentSlug = 'alpha'
   serviceGitBranch = 'master'
   serviceGitRepo = "https://freeleaps@dev.azure.com/freeleaps/magicleaps/_git/magicleaps"
-  environmentSlug = 'alpha'
-  dependenciesManager = 'pip'
-  pipRequirementsFile = 'requirements.txt'
-  buildCacheEnabled = true
+  serviceGitRepoType = 'monorepo'
+  executeMode = 'on-demand' // on-demand, full
+  commitMessageLintEnabled = true
+  components {
+    frontend {
+      root = 'frontend'
+      language = 'javascript'
+      dependenciesManager = 'npm'
+      buildAgentImage = 'node:lts-alpine'
+      buildCacheEnabled = true
+      buildCommand = 'npm run build'
+      lintEnabled = true
+      linter = 'eslint'
+      sastEnabled = true
+      sastProvider = 'NodeJsScan'
+      imageRegistry = 'docker.io'
+      imageRepository = 'sunzhenyucn'
+      imageName = 'magicleaps-frontend'
+      imageBuilder = 'dind'
+      dockerfilePath = 'Dockerfile'
+      imageBuildRoot = '.'
+      imageReleaseArchitectures = ['amd64', 'arm64']
+      registryCredentialName = 'first-class-pipeline-dev-secret'
+      semanticReleaseEnabled = true
+      semanticReleaseBranch = 'master'
+    }
+
+    backend {
+      root = 'backend'
+      language = 'python'
+      dependenciesManager = 'pip'
+      buildAgentImage = 'python:3.10-slim-buster'
+      buildCacheEnabled = true
+      lintEnabled = true
+      linter = 'PyLint'
+      sastEnabled = true
+      sastProvider = 'Bandit'
+      imageRegistry = 'docker.io'
+      imageRepository = 'sunzhenyucn'
+      imageName = 'magicleaps-backend'
+      imageBuilder = 'dind'
+      dockerfilePath = 'Dockerfile'
+      imageBuildRoot = '.'
+      imageReleaseArchitectures = ['amd64', 'arm64']
+      registryCredentialName = 'first-class-pipeline-dev-secret'
+      semanticReleaseEnabled = true
+      semanticReleaseBranch = 'master'
+    }
+  }
 }
\ No newline at end of file
diff --git a/first-class-pipeline/vars/executeFreeleapsPipeline.groovy b/first-class-pipeline/vars/executeFreeleapsPipeline.groovy
index 1174dbfe..dfb383b6 100644
--- a/first-class-pipeline/vars/executeFreeleapsPipeline.groovy
+++ b/first-class-pipeline/vars/executeFreeleapsPipeline.groovy
@@ -4,6 +4,8 @@ import com.freeleaps.devops.SourceFetcher
 import com.freeleaps.devops.DependenciesResolver
 import com.freeleaps.devops.enums.DependenciesManager
 import com.freeleaps.devops.enums.ServiceLanguage
+import com.freeleaps.devops.CommitMessageLinter
+import com.freeleaps.devops.ChangedComponentsDetector
 
 def call(body) {
   def configurations = [:]
@@ -11,6 +13,8 @@ def call(body) {
   body.delegate = configurations
   body()
 
+  def sourceFetcher = new SourceFetcher(this)
+
   pipeline {
     agent any
     options {
@@ -19,59 +23,131 @@ def call(body) {
     }
 
     stages {
-
-      stage("Source Codes Checkout") {
-        steps {
-          script {
-            def sourceFetcher = new SourceFetcher(this)
-            sourceFetcher.fetch(configurations)
+      stage("Commit Linting If Enabled") {
+        agent {
+          kubernetes {
+            defaultContainer 'commit-message-linter'
+            yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    freeleaps-devops-system/milestone: commit-message-linting
+spec:
+  containers:
+  - name: commit-message-linter
+    image: docker.io/commitlint/commitlint:master
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: workspace
+      mountPath: /workspace
+  volumes:
+    - name: workspace
+      emptyDir: {} 
+"""
           }
         }
-      }
-
-      stage("Commit Linting If Enabled") {
         steps {
           script {
             def enabled = configurations.commitMessageLintEnabled
             
             if (enabled == null || !enabled) {
               log.warn "Commit message linting is disabled"
+            } else if (enabled) {
+              log.info "Commit message linting is enabled"
+              sourceFetcher.fetch(configurations)
+
+              def linter = new CommitMessageLinter(this)
+              linter.lint(configurations)
             }
           }
         }
       }
 
-      stage("Build Agent Setup") {
+      stage("Execute Mode Detection") {
         steps {
           script {
-            def buildAgentImage = configurations.buildAgentImage
-            if (buildAgentImage == null || buildAgentImage.isEmpty()) {
-              log.warn "Not set buildAgentImage, using default build agent image"
-
-              def language = ServiceLanguage.parse(configurations.serviceLang)
-              switch(language) {
-                case ServiceLanguage.PYTHON:
-                  buildAgentImage = "python:3.10-slim-buster"
-                  break
-                case ServiceLanguage.NODE:
-                  buildAgentImage = "node:lts-alpine"
-                  break
-                default:
-                  error("Unknown service language")
-              }
-
-              log.info "Using ${buildAgentImage} as build agent image"
-              env.buildAgentImage = buildAgentImage
+            def executeMode = configurations.executeMode
+            if (executeMode == null || executeMode.isEmpty()) {
+              log.warn "Not set executeMode, using fully as default execute mode"
+              env.executeMode = "fully"
+            } else if (executeMode == 'on-demand' && serviceGitRepoType != 'monorepo') {
+              log.warn "serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode"
+              env.executeMode = "fully"
+            } else {
+              log.info "Using ${executeMode} as execute mode"
+              env.executeMode = executeMode
             }
           }
         }
       }
 
-      stage("Dependencies Resolving") {
-        agent {
-          kubernetes {
-            defaultContainer 'dep-resolver'
-            yaml """
+      stage("Code Changes Detection") {
+        steps {
+          when {
+            expression {
+              return env.executeMode == "on-demand"
+            }
+          }
+
+          script {
+            sourceFetcher.fetch(configurations)
+
+            def changedComponentsDetector = new ChangedComponentsDetector(this)
+            def changedComponents = changedComponentsDetector.detect(env.workspace, configurations.components)
+
+            log.info "Changed components: ${changedComponents}"
+            env.changedComponents = changedComponents.join(' ')
+          }
+        }
+      }
+
+      configurations.components.each { component ->
+        stage("${component} :: Build Agent Setup") {
+          when {
+            expression {
+              return env.executeMode == "fully" || env.changedComponents.contains(component)
+            }
+          }
+
+          steps {
+            script {
+              def buildAgentImage = component.buildAgentImage
+              if (buildAgentImage == null || buildAgentImage.isEmpty()) {
+                log.warn "Not set buildAgentImage for ${component}, using default build agent image"
+
+                def language = ServiceLanguage.parse(configurations.serviceLang)
+                switch(language) {
+                  case ServiceLanguage.PYTHON:
+                    buildAgentImage = "python:3.10-slim-buster"
+                    break
+                  case ServiceLanguage.JS:
+                    buildAgentImage = "node:lts-alpine"
+                    break
+                  default:
+                    error("Unknown service language")
+                }
+
+                log.info "Using ${buildAgentImage} as build agent image for ${component}"
+                env.buildAgentImage = buildAgentImage
+              }
+            }
+          }
+        }
+
+        stage("${component} :: Dependencies Resolving") {
+          when {
+            expression {
+              return env.executeMode == "fully" || env.changedComponents.contains(component)
+            }
+          }
+
+          agent {
+            kubernetes {
+              defaultContainer 'dep-resolver'
+              yaml """
 apiVersion: v1
 kind: Pod
 metadata:
@@ -89,26 +165,28 @@ spec:
       mountPath: /workspace
   volumes:
     - name: workspace
-      emptyDir: {} 
+      emptyDir: {}
 """
-          }
-        }
-        steps {
-          script {
-            def language = ServiceLanguage.parse(configurations.serviceLang)
-
-            def depManager = DependenciesManager.parse(configurations.dependenciesManager)
-
-            def dependenciesResolver = new DependenciesResolver(this, language)
-            dependenciesResolver.useManager(depManager)
-            
-            if (configurations.buildCacheEnabled) {
-              dependenciesResolver.enableCachingSupport()
-            } else {
-              dependenciesResolver.disableCachingSupport()
             }
+          }
 
-            dependenciesResolver.resolve(configurations)
+          steps {
+            script {
+              def language = ServiceLanguage.parse(component.language)
+
+              def depManager = DependenciesManager.parse(component.dependenciesManager)
+
+              def dependenciesResolver = new DependenciesResolver(this, language)
+              dependenciesResolver.useManager(depManager)
+              
+              if (component.buildCacheEnabled) {
+                dependenciesResolver.enableCachingSupport()
+              } else {
+                dependenciesResolver.disableCachingSupport()
+              }
+
+              dependenciesResolver.resolve(configurations)
+            }
           }
         }
       }