refactor(SourceFetcher, ServiceLanguage, DependenciesResolver): improve workspace handling, standardize language naming, and streamline dependency resolution

Signed-off-by: 孙振宇 <>

first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js

@@ -0,0 +1,30 @@
+module.exports = {
+  extends: ["@commitlint/config-angular"], // Extends with the Angular commitlint configuration
+  rules: {
+    "type-enum": [
+      2,
+      "always",
+      [
+        "feat",
+        "fix",
+        "docs",
+        "style",
+        "refactor",
+        "perf",
+        "test",
+        "build",
+        "ci",
+        "chore",
+        "revert", // Add or remove types as needed
+      ],
+    ],
+    "type-case": [2, "always", "lower-case"], // Type must be in lower case
+    "type-empty": [2, "never"], // Type must not be empty
+    "scope-empty": [2, "never"], // Scope must not be empty
+    "scope-case": [2, "always", "lower-case"], // Scope must be in lower case
+    "subject-empty": [2, "never"], // Subject must not be empty
+    "subject-case": [2, "never", []], // Subject must be in sentence case
+    "subject-full-stop": [2, "never", "."], // Subject must not end with a period
+    "header-max-length": [2, "always", 100], // Header must not exceed 100 characters
+  },
+};

first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy

@@ -0,0 +1,31 @@
+package com.freeleaps.devops
+
+class ChangedComponentsDetector {
+  def steps
+  def workspace
+
+  ChangedComponentsDetector(steps) {
+    this.steps = steps
+  }
+
+  def detect(workspace, components) {
+    def changedComponents = [] as Set
+
+    dir(workspace) {
+      // using git command to get changed files list
+      def changedFiles = sh(script: 'git diff --name-only HEAD~1 HEAD', returnStdout: true)
+        .trim()
+        .split('\n')
+      
+      changedFiles.each { file ->
+        components.each { component ->
+          if (file.startsWith("${component}/")) {
+            changedComponents.add(component)
+          }
+        }
+      }
+    }
+
+    return changedComponents.toList()
+  }
+}

first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy

@@ -0,0 +1,28 @@
+package com.freeleaps.devops
+
+class CommitMessageLinter {
+  def steps
+  private defaultRule = 'com/freeleaps/devops/builtins/commitlint/default.js'
+
+  CommitMessageLinter(steps) {
+    this.steps = steps
+  }
+
+  def lint(configurations) {
+    def rules = steps.libraryResource 'com/freeleaps/devops/builtins/commitlint/default.js'
+    steps.log.info "Check if there has custom commit lint rules specified..."
+
+    if (configurations.commitLintRules != null && !configurations.commitLintRules.isEmpty()) {
+      steps.log.info "Custom commit lint rules found, using custom rules files: ${configurations.commitLintRules}"
+      rules = configurations.commitLintRules
+    } else {
+      steps.log.info "No custom commit lint rules found, using built-in rules at: ${defaultRules}"
+      steps.sh "echo ${rules} > .commitlintrc.js"
+      rules = '.commitlintrc.js'
+    }
+
+    steps.log.info "Linting commit messages from HEAD..."
+
+    steps.sh "commitlint --verbose -g ${rules} -f HEAD^"
+  }
+}

first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy

@@ -43,19 +43,8 @@ class DependenciesResolver {
 
     switch (mgr) {
       case DependenciesManager.PIP:
-        if (configurations.pipRequirementsFile == null || configurations.pipRequirementsFile.isEmpty()) {
+        steps.log.warn "Python project no need to resolving dependencies, skipping..."
-          steps.error("pipRequirementsFile is required when using PIP as dependencies manager")
+        break
-        }
-
-        def requirementsFile = configurations.pipRequirementsFile
-
-        if (cachingEnabled) {
-          steps.cache(maxCacheSize: 512, caches: [[$class: 'ArbitraryFileCache', excludes: '', includes: '**/*', path: '.pip-cache']]) {
-            steps.sh "pip install -r ${requirementsFile} --cache-dir .pip-cache"
-          }
-        } else {
-          steps.sh "pip install -r ${requirementsFile}"
-        }
       case DependenciesManager.NPM:
         if (configurations.npmPackageJsonFile == null || configurations.npmPackageJsonFile.isEmpty()) {
           steps.error("npmPackageJsonFile is required when using NPM as dependencies manager")
@@ -70,6 +59,8 @@ class DependenciesResolver {
         } else {
           steps.sh "npm install"
         }
+
+        break
       case DependenciesManager.YARN:
         if (configurations.yarnPackageJsonFile == null || configurations.yarnPackageJsonFile.isEmpty()) {
           steps.error("yarnPackageJsonFile is required when using YARN as dependencies manager")
@@ -84,6 +75,8 @@ class DependenciesResolver {
         } else {
           steps.sh "yarn install"
         }
+
+        break
       default:
         steps.error("Unsupported dependencies manager")
     }

first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy

@@ -16,6 +16,10 @@ class SourceFetcher {
       steps.error("serviceGitBranch is required")
     }
 
+    steps.env.workspace = "workspace"
+
+    dir(steps.env.workspace) {
       steps.git branch: configurations.serviceGitBranch, credentialsId: 'git-bot-credentials', url: configurations.serviceGitRepo
     }
   }
+}

first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy

@@ -2,8 +2,8 @@ package com.freeleaps.devops.enums
 
 enum ServiceLanguage {
 
-  PYTHON('Python'),
+  PYTHON('python'),
-  NODE('Node (JS,TS)'),
+  JS('javascript'),
   UNKNOWN('Unknown')
 
   final String language
@@ -14,10 +14,10 @@ enum ServiceLanguage {
 
   static ServiceLanguage parse(String language) {
     switch (language) {
-      case 'Python':
+      case 'python':
         return ServiceLanguage.PYTHON
-      case 'Node (JS,TS)':
+      case 'javascript':
-        return ServiceLanguage.NODE
+        return ServiceLanguage.JS
       default:
         return ServiceLanguage.UNKNOWN
     }

first-class-pipeline/tests/Jenkinsfile

@@ -2,11 +2,56 @@ library 'first-class-pipeline'
 
 executeFreeleapsPipeline {
   serviceName = 'magicleaps'
-  serviceLang = 'Python'
+  environmentSlug = 'alpha'
   serviceGitBranch = 'master'
   serviceGitRepo = "https://freeleaps@dev.azure.com/freeleaps/magicleaps/_git/magicleaps"
-  environmentSlug = 'alpha'
+  serviceGitRepoType = 'monorepo'
-  dependenciesManager = 'pip'
+  executeMode = 'on-demand' // on-demand, full
-  pipRequirementsFile = 'requirements.txt'
+  commitMessageLintEnabled = true
+  components {
+    frontend {
+      root = 'frontend'
+      language = 'javascript'
+      dependenciesManager = 'npm'
+      buildAgentImage = 'node:lts-alpine'
       buildCacheEnabled = true
+      buildCommand = 'npm run build'
+      lintEnabled = true
+      linter = 'eslint'
+      sastEnabled = true
+      sastProvider = 'NodeJsScan'
+      imageRegistry = 'docker.io'
+      imageRepository = 'sunzhenyucn'
+      imageName = 'magicleaps-frontend'
+      imageBuilder = 'dind'
+      dockerfilePath = 'Dockerfile'
+      imageBuildRoot = '.'
+      imageReleaseArchitectures = ['amd64', 'arm64']
+      registryCredentialName = 'first-class-pipeline-dev-secret'
+      semanticReleaseEnabled = true
+      semanticReleaseBranch = 'master'
+    }
+
+    backend {
+      root = 'backend'
+      language = 'python'
+      dependenciesManager = 'pip'
+      buildAgentImage = 'python:3.10-slim-buster'
+      buildCacheEnabled = true
+      lintEnabled = true
+      linter = 'PyLint'
+      sastEnabled = true
+      sastProvider = 'Bandit'
+      imageRegistry = 'docker.io'
+      imageRepository = 'sunzhenyucn'
+      imageName = 'magicleaps-backend'
+      imageBuilder = 'dind'
+      dockerfilePath = 'Dockerfile'
+      imageBuildRoot = '.'
+      imageReleaseArchitectures = ['amd64', 'arm64']
+      registryCredentialName = 'first-class-pipeline-dev-secret'
+      semanticReleaseEnabled = true
+      semanticReleaseBranch = 'master'
+    }
+  }
 }

first-class-pipeline/vars/executeFreeleapsPipeline.groovy

@@ -4,6 +4,8 @@ import com.freeleaps.devops.SourceFetcher
 import com.freeleaps.devops.DependenciesResolver
 import com.freeleaps.devops.enums.DependenciesManager
 import com.freeleaps.devops.enums.ServiceLanguage
+import com.freeleaps.devops.CommitMessageLinter
+import com.freeleaps.devops.ChangedComponentsDetector
 
 def call(body) {
   def configurations = [:]
@@ -11,6 +13,8 @@ def call(body) {
   body.delegate = configurations
   body()
 
+  def sourceFetcher = new SourceFetcher(this)
+
   pipeline {
     agent any
     options {
@@ -19,55 +23,127 @@ def call(body) {
     }
 
     stages {
-
-      stage("Source Codes Checkout") {
-        steps {
-          script {
-            def sourceFetcher = new SourceFetcher(this)
-            sourceFetcher.fetch(configurations)
-          }
-        }
-      }
-
       stage("Commit Linting If Enabled") {
+        agent {
+          kubernetes {
+            defaultContainer 'commit-message-linter'
+            yaml """
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    freeleaps-devops-system/milestone: commit-message-linting
+spec:
+  containers:
+  - name: commit-message-linter
+    image: docker.io/commitlint/commitlint:master
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: workspace
+      mountPath: /workspace
+  volumes:
+    - name: workspace
+      emptyDir: {} 
+"""
+          }
+        }
         steps {
           script {
             def enabled = configurations.commitMessageLintEnabled
             
             if (enabled == null || !enabled) {
               log.warn "Commit message linting is disabled"
+            } else if (enabled) {
+              log.info "Commit message linting is enabled"
+              sourceFetcher.fetch(configurations)
+
+              def linter = new CommitMessageLinter(this)
+              linter.lint(configurations)
             }
           }
         }
       }
 
-      stage("Build Agent Setup") {
+      stage("Execute Mode Detection") {
         steps {
           script {
-            def buildAgentImage = configurations.buildAgentImage
+            def executeMode = configurations.executeMode
+            if (executeMode == null || executeMode.isEmpty()) {
+              log.warn "Not set executeMode, using fully as default execute mode"
+              env.executeMode = "fully"
+            } else if (executeMode == 'on-demand' && serviceGitRepoType != 'monorepo') {
+              log.warn "serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode"
+              env.executeMode = "fully"
+            } else {
+              log.info "Using ${executeMode} as execute mode"
+              env.executeMode = executeMode
+            }
+          }
+        }
+      }
+
+      stage("Code Changes Detection") {
+        steps {
+          when {
+            expression {
+              return env.executeMode == "on-demand"
+            }
+          }
+
+          script {
+            sourceFetcher.fetch(configurations)
+
+            def changedComponentsDetector = new ChangedComponentsDetector(this)
+            def changedComponents = changedComponentsDetector.detect(env.workspace, configurations.components)
+
+            log.info "Changed components: ${changedComponents}"
+            env.changedComponents = changedComponents.join(' ')
+          }
+        }
+      }
+
+      configurations.components.each { component ->
+        stage("${component} :: Build Agent Setup") {
+          when {
+            expression {
+              return env.executeMode == "fully" || env.changedComponents.contains(component)
+            }
+          }
+
+          steps {
+            script {
+              def buildAgentImage = component.buildAgentImage
               if (buildAgentImage == null || buildAgentImage.isEmpty()) {
-              log.warn "Not set buildAgentImage, using default build agent image"
+                log.warn "Not set buildAgentImage for ${component}, using default build agent image"
 
                 def language = ServiceLanguage.parse(configurations.serviceLang)
                 switch(language) {
                   case ServiceLanguage.PYTHON:
                     buildAgentImage = "python:3.10-slim-buster"
                     break
-                case ServiceLanguage.NODE:
+                  case ServiceLanguage.JS:
                     buildAgentImage = "node:lts-alpine"
                     break
                   default:
                     error("Unknown service language")
                 }
 
-              log.info "Using ${buildAgentImage} as build agent image"
+                log.info "Using ${buildAgentImage} as build agent image for ${component}"
                 env.buildAgentImage = buildAgentImage
               }
             }
           }
         }
 
-      stage("Dependencies Resolving") {
+        stage("${component} :: Dependencies Resolving") {
+          when {
+            expression {
+              return env.executeMode == "fully" || env.changedComponents.contains(component)
+            }
+          }
+
           agent {
             kubernetes {
               defaultContainer 'dep-resolver'
@@ -93,16 +169,17 @@ spec:
 """
             }
           }
+
           steps {
             script {
-            def language = ServiceLanguage.parse(configurations.serviceLang)
+              def language = ServiceLanguage.parse(component.language)
 
-            def depManager = DependenciesManager.parse(configurations.dependenciesManager)
+              def depManager = DependenciesManager.parse(component.dependenciesManager)
 
               def dependenciesResolver = new DependenciesResolver(this, language)
               dependenciesResolver.useManager(depManager)
               
-            if (configurations.buildCacheEnabled) {
+              if (component.buildCacheEnabled) {
                 dependenciesResolver.enableCachingSupport()
               } else {
                 dependenciesResolver.disableCachingSupport()
@@ -112,6 +189,7 @@ spec:
             }
           }
         }
+      }
 
     }
   }