freeleaps-ops/first-class-pipeline/vars/executeFreeleapsPipeline.groovy

#!groovy

import com.freeleaps.devops.SourceFetcher
import com.freeleaps.devops.DependenciesResolver
import com.freeleaps.devops.CommitMessageLinter
import com.freeleaps.devops.ChangedComponentsDetector
import com.freeleaps.devops.CodeLintExecutor
import com.freeleaps.devops.SASTExecutor
import com.freeleaps.devops.ImageBuilder
import com.freeleaps.devops.SemanticReleasingExecutor
import com.freeleaps.devops.ArgoApplicationVersionUpdater

import com.freeleaps.devops.enums.DependenciesManager
import com.freeleaps.devops.enums.ServiceLanguage
import com.freeleaps.devops.enums.CodeLinterTypes
import com.freeleaps.devops.enums.ImageBuilderTypes
import org.csanchez.jenkins.plugins.kubernetes.pod.yaml.YamlMergeStrategy
import org.csanchez.jenkins.plugins.kubernetes.pod.yaml.Merge


def generateComponentStages(component, configurations) {
  def stages = []
  stages.addAll([
    // Build Agent Setup
    {stage("${component.name} :: Build Agent Setup") {
      script {
          if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
              def buildAgentImage = component.buildAgentImage
              if (buildAgentImage == null || buildAgentImage.isEmpty()) {
                  log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image")

                  def language = ServiceLanguage.parse(component.language)
                  switch(language) {
                      case ServiceLanguage.PYTHON:
                          buildAgentImage = "docker.io/python:3.10-slim-buster"
                          break
                      case ServiceLanguage.JS:
                          buildAgentImage = "docker.io/node:lts-alpine"
                          break
                      default:
                          error("Unknown service language")
                  }
              }
              log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}")
              env."${component.name}_buildAgentImage" = buildAgentImage
          }
      }
    }},
    // Dependencies Resolving
    {stage("${component.name} :: Dependencies Resolving") {
      podTemplate(
        label: "dep-resolver-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
        containers: [
          containerTemplate(
            name: 'dep-resolver',
            image: env."${component.name}_buildAgentImage",
            ttyEnabled: true,
            command: 'sleep',
            args: 'infinity'
          )
        ]
      ) {
        node("dep-resolver-${component.name}") {
          container('dep-resolver') {
            script {
              if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                def buildAgentImage = env."${component.name}_buildAgentImage"
                log.info("Pipeline", "Using ${buildAgentImage} as build agent image for dependencies resolving")
                def sourceFetcher = new SourceFetcher(this)
                sourceFetcher.fetch(configurations)

                def language = ServiceLanguage.parse(component.language)
                def depManager = DependenciesManager.parse(component.dependenciesManager)

                def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
                dependenciesResolver.useManager(depManager)
                
                if (component.buildCacheEnabled) {
                    dependenciesResolver.enableCachingSupport()
                } else {
                    dependenciesResolver.disableCachingSupport()
                }

                dependenciesResolver.resolve(component)
              }
            }
          }
        }
      }
    }},
  ])

  if (component.lintEnabled != null && component.lintEnabled) {
    stages.addAll([
      // Code Linter Environment Preparation
      {stage("${component.name} :: Code Linter Preparation") {
        script {
          if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
            if (component.lintEnabled != null && component.lintEnabled) {
              log.info("Pipeline", "Code linting has enabled, preparing linter...")

              if (component.linter == null || component.linter.isEmpty()) {
                log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback")
              }

              def linter = CodeLinterTypes.parse(component.linter)

              if (linter == null) {
                log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting")
              }

              if (linter.language != ServiceLanguage.parse(component.language)) {
                log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting")
              }

              log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}")
              env."${component.name}_linterContainerImage" = linter.containerImage
            } else {
              log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...")
            }
          }
        }
      }},
      // Code Linting
      {stage("${component.name} :: Code Linting") {
        podTemplate(
          label: "code-linter-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
          containers: [
            containerTemplate(
              name: 'code-linter',
              image: env."${component.name}_linterContainerImage",
              ttyEnabled: true,
              command: 'sleep',
              args: 'infinity'
            )
          ]
        ) {
          node("code-linter-${component.name}") {
            container('code-linter') {
              script {
                if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                  if (component.lintEnabled != null && component.lintEnabled) {
                    log.info("Pipeline", "Code linting has enabled, linting code...")

                    def sourceFetcher = new SourceFetcher(this)
                    sourceFetcher.fetch(configurations)

                    def linterType = CodeLinterTypes.parse(component.linter)
                    def language = ServiceLanguage.parse(component.language)

                    def depManager = DependenciesManager.parse(component.dependenciesManager)

                    // resolving deps from cache
                    def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
                    dependenciesResolver.useManager(depManager)
                    
                    if (component.buildCacheEnabled) {
                        dependenciesResolver.enableCachingSupport()
                    } else {
                        dependenciesResolver.disableCachingSupport()
                    }

                    dependenciesResolver.resolve(component)

                    // lint codes
                    def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType, component)
                    codeLintExecutor.execute()
                  }
                }
              }
            }
          }
        }
      }}
    ])
  }

  if (component.sastEnabled != null && component.sastEnabled) {
    stages.addAll([
      // SAST Scanner Environment Preparation
      {stage("${component.name} :: SAST Scanner Preparation") {
        script {
          if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
            if (component.sastEnabled != null && component.sastEnabled) {
              log.info("Pipeline", "SAST scanning has enabled, preparing scanner...")

              if (sastScanner == null || sastScanner.isEmpty()) {
                log.error("Pipeline", "Not set sastScanner for ${component.name}")
              }

              def sastScannerType = SASTScannerTypes.parse(component.sastScanner)
              if (sastScannerType == null) {
                log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning")
              } else if (sastScannerType.language != ServiceLanguage.parse(component.language)) {
                log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning")
              } else {
                log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}")
                env."${component.name}_sastScannerContainerImage" = sastScannerType.containerImage
              }
            }
          }
        }
      }},
      // SAST Scanning
      {stage("${component.name} :: SAST Scanning") {
        when {
          expression {
            return (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty()
          }
        }
        podTemplate(
          label: "sast-scanner-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
          containers: [
            containerTemplate(
              name: 'sast-scanner',
              image: env."${component.name}_sastScannerContainerImage",
              ttyEnabled: true,
              command: 'sleep',
              args: 'infinity'
            )
          ]
        ) {
          node("sast-scanner-${component.name}") {
            container('sast-scanner') {
              script {
                if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                  if (component.sastEnabled != null && component.sastEnabled) {
                    log.info("Pipeline", "SAST scanning has enabled, scanning code...")

                    def sourceFetcher = new SourceFetcher(this)
                    sourceFetcher.fetch(configurations)

                    def sastScannerType = SASTScannerTypes.parse(component.sastScanner)

                    def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType)
                    sastScanner.scan()
                  }
                }
              }
            }
          }
        }
      }}
    ])
  }

  if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) {
    stages.addAll([
      // Semantic Releasing
      {stage("${component.name} :: Semantic Releasing") {
        podTemplate(
          label: "semantic-releasing-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
          containers: [
            containerTemplate(
              name: 'semantic-releasing',
              image: 'node:18-bullseye-slim',
              ttyEnabled: true,
              command: 'sleep',
              args: 'infinity'
            )
          ]
        ) {
          node("semantic-releasing-${component.name}") {
            container('semantic-releasing') {
              script {
                if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                  if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) {
                    log.info("Pipeline", "Semantic releasing has enabled, releasing...")

                    if (env.SEMANTIC_RELEASED != null && !env.SEMANTIC_RELEASED.isEmpty() && env.SEMANTIC_RELEASED) {
                      log.info("Pipeline", "Semantic release has been executed, skipping...")
                      return
                    }

                    def sourceFetcher = new SourceFetcher(this)
                    sourceFetcher.fetch(configurations)

                    def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot)
                    semanticReleasingExecutor.release(configurations.serviceGitCredentialsId, configurations.serviceGitBranch)
                  }
                }
              }
            }
          }
        }
      }}
    ])
  }

  stages.addAll([
    // Compilation & Packaging
    {stage("${component.name} :: Compilation & Packaging") {
      podTemplate(
        label: "build-agent-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
        containers: [
          containerTemplate(
            name: 'build-agent',
            image: env."${component.name}_buildAgentImage",
            ttyEnabled: true,
            command: 'sleep',
            args: 'infinity',
            resourceLimitCpu: "1",
            resourceLimitMemory: "2Gi",
            resourceRequestCpu: "0.5",
            resourceRequestMemory: "1Gi"
          )
        ]
      ) {
        node("build-agent-${component.name}") {
          container('build-agent') {
            script {
              if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                def buildAgentImage = env."${component.name}_buildAgentImage"
                log.info("Pipeline", "Using ${buildAgentImage} as build agent image for compilation & packaging")
                
                def sourceFetcher = new SourceFetcher(this)
                sourceFetcher.fetch(configurations)

                def language = ServiceLanguage.parse(component.language)
                def depManager = DependenciesManager.parse(component.dependenciesManager)

                def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
                dependenciesResolver.useManager(depManager)
                if (component.buildCacheEnabled) {
                    dependenciesResolver.enableCachingSupport()
                } else {
                    dependenciesResolver.disableCachingSupport()
                }

                dependenciesResolver.resolve(component)

                dir(env.workroot + "/" + component.root) {
                  if (component.buildCommand != null && !component.buildCommand.isEmpty()) {
                    sh component.buildCommand
                  }
                  component.buildArtifacts.each { artifact ->
                    log.info("Pipeline", "Stashing artifact ${artifact} for ${component.name}...")
                    def artifactList = sh(script: "ls ${artifact} -al", returnStdout: true)
                    log.info("Pipeline", "Artifacts list: ${artifactList}")
                    def targetPathType = sh(
                      script: """
                        if [ -d "${artifact}" ]; then
                          echo "dir"
                        elif [ -f "${artifact}" ]; then
                          echo "file"
                        else
                          echo "unknown"
                        fi
                      """,
                      returnStdout: true
                    )
                    if (artifact == '.' || artifact == './') {
                      log.info("Pipeline", "Stashing root directory for ${component.name}...")
                      stash includes: "", name: "${component.name}-root"
                    } else if (targetPathType.trim() == "dir") {
                      stash includes: "${artifact}/**", name: "${component.name}-${artifact}"
                    } else {
                      stash includes: artifact, name: "${component.name}-${artifact}"
                    }
                  }
                }
              }
            }
          }
        }
      }
    }},
    // Image Builder Setup
    {stage("${component.name} :: Image Builder Setup") {
      script {
        if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
          log.info("Pipeline", "Ready to setup image builder for ${component.name}")
          def imageBuilder
          if (component.imageBuilder == null || component.imageBuilder.isEmpty()) {
            log.info("Pipeline", "imageBuilder not set for ${component.name}, using kaniko as default image builder")
            imageBuilder = ImageBuilderTypes.KANIKO
          } else {
            imageBuilder = ImageBuilderTypes.parse(component.imageBuilder)
            if (imageBuilder == null) {
              log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building")
            }
          }

          env."${component.name}_imageBuilderImage" = imageBuilder.image
          log.info("Pipeline", "Using ${imageBuilder.builder} (image: ${imageBuilder.image}) as image builder for ${component.name}")
        }
      }
    }},
    // Image Building & Publishing
    {stage("${component.name} :: Image Building & Publishing") {
      podTemplate(
        label: "image-builder-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
        containers: [
          containerTemplate(
            name: 'image-builder',
            image: env."${component.name}_imageBuilderImage",
            privileged: true,
            ttyEnabled: true,
            command: 'sleep',
            args: 'infinity'
          )
        ],
        volumes: [
          hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock')
        ]
      ) {
        node("image-builder-${component.name}") {
          container('image-builder') {
            script {
              if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                def sourceFetcher = new SourceFetcher(this)
                sourceFetcher.fetch(configurations)

                dir(env.workroot + "/" + component.root) {
                  component.buildArtifacts.each { artifact ->
                    if (artifact == '.' || artifact == './') {
                      unstash "${component.name}-root"
                    } else {
                      unstash "${component.name}-${artifact}"
                    }
                  }

                  if (component.dockerfile != null && !component.dockerfile.isEmpty()) {
                    log.error("Pipeline", "Component ${component.name} dockerfile not set!")
                  }

                  def imageBuilderType = ImageBuilderTypes.parse(component.imageBuilder)
                  if (imageBuilderType == null) {
                    log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building")
                  }

                  def imageBuilder = new ImageBuilder(this, 
                    env.workroot + "/" + component.root + "/",
                    component.imageBuildRoot, 
                    component.dockerfilePath, 
                    imageBuilderType
                  )

                  log.info("Pipeline", "Retrieve version of image from pervious stage...")
                  if (env.LATEST_VERSION == null || env.LATEST_VERSION.isEmpty()) {
                    log.warn("Pipeline", "LATEST_VERSION environment value not set, using 'snapshot-<BUILD_COMMIT_HASH>' as default version")
                    sh "git config --global --add safe.directory ${env.workroot}"
                    def commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
                    def shortCommitHash = commitHash.take(7)
                    env.LATEST_VERSION = "snapshot-${shortCommitHash}"
                  }
                  def version

                  imageBuilder.setManifestsOfImage(component.imageRegistry, component.imageRepository, component.imageName, env.LATEST_VERSION)
                  imageBuilder.useCredentials(component.registryCredentialsId)
                  imageBuilder.setArchitectures(component.imageReleaseArchitectures)
                  imageBuilder.build()
                }               
              }
            }
          }
        }
      }
    }},
    {stage("${component.name} :: Argo Application Version Updating") {
      podTemplate(
        label: "argo-app-version-updater-${component.name}",
        yaml: """
apiVersion: v1
kind: Pod
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
        """,
        yamlMergeStrategy: merge(),
        containers: [
          containerTemplate(
            name: "argo-app-version-updater",
            image: "debian:12",
            ttyEnabled: true,
            command: 'sleep',
            args: 'infinity'
          )
        ]
      ) {
        node("argo-app-version-updater-${component.name}") {
          container("argo-app-version-updater") {
            script {
              if (env.executeMode == "fully" || env.changedComponents.split(/\s+/).toList().contains(component.name)) {
                def argoApplicationVersionUpdater = new ArgoApplicationVersionUpdater(this, configurations)
                argoApplicationVersionUpdater.update(configurations.environmentSlug, component)
              }
            }
          }
        }
      }
    }}
  ])
  return {
    stages.each { stageClosure ->
      stageClosure()
    }
  }
}

def call(Closure closure) {
  def configurations = [:]
  closure.resolveStrategy = Closure.DELEGATE_FIRST
  closure.delegate = configurations
  closure()

  pipeline {
    agent any
    options {
      buildDiscarder(logRotator(numToKeepStr: '25'))
      timeout(time: 30, unit: 'MINUTES')
      parallelsAlwaysFailFast()
    }

    stages {
      stage("Pipeline :: Commit Linting If Enabled") {
        when {
          expression {
            return configurations.commitMessageLintEnabled != null && configurations.commitMessageLintEnabled
          }
        }
        agent {
          kubernetes {
            defaultContainer 'commit-message-linter'
            yaml """
apiVersion: v1
kind: Pod
metadata:
  labels:
    freeleaps-devops-system/milestone: commit-message-linting
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: "node-role.kubernetes.io/devops"
                operator: In
                values: ["true"]
  tolerations:
    - key: "node-role.kubernetes.io/devops"
      operator: Equal
      value: "true"
      effect: "NoSchedule"
  containers:
  - name: commit-message-linter
    image: docker.io/commitlint/commitlint:master
    command:
    - cat
    tty: true
    volumeMounts:
    - name: workspace
      mountPath: /workspace
  volumes:
    - name: workspace
      emptyDir: {} 
"""
          }
        }
        steps {
          script {
            log.info("Pipeline","Commit message linting is enabled")
            def sourceFetcher = new SourceFetcher(this)
            sourceFetcher.fetch(configurations)

            def linter = new CommitMessageLinter(this)
            linter.lint(configurations)
          }
        }
      }

      stage("Pipeline :: Execute Mode Detection") {
        steps {
          script {
            def executeMode = configurations.executeMode
            if (executeMode == null || executeMode.isEmpty()) {
              log.warn("Pipeline","Not set executeMode, using fully as default execute mode")
              env.executeMode = "fully"
            } else if (executeMode == 'on-demand' && configurations.serviceGitRepoType != 'monorepo') {
              log.warn("Pipeline","serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode")
              env.executeMode = "fully"
            } else {
              log.info("Pipeline","Using ${executeMode} as execute mode")
              env.executeMode = executeMode
            }
          }
        }
      }

      stage("Pipeline :: Code Changes Detection") {
        when {
          expression {
            return env.executeMode == "on-demand"
          }
        }

        steps {
          script {
            def sourceFetcher = new SourceFetcher(this)
            sourceFetcher.fetch(configurations)

            def changedComponentsDetector = new ChangedComponentsDetector(this)
            def changedComponents = changedComponentsDetector.detect(env.workroot, configurations.components)

            log.info("Pipeline","Changed components: ${changedComponents}")
            env.changedComponents = changedComponents.join(' ')
          }
        }
      }

      stage("Pipeline :: Components Build (Dynamic Generated Stages)") {
        steps {
          script {
            if (env.executeMode == "fully") {
              configurations.components.each { component ->
                log.info("Pipeline", "Executing generated stages for ${component.name}...")
                generateComponentStages(component, configurations)()
              }
            } else if {
              def changedComponents = env.changedComponents.split(/\s+/).toList()
              configurations.components.each { component ->
                if (changedComponents.contains(component.name)) {
                  log.info("Pipeline", "Executing generated stages for ${component.name}...")
                  generateComponentStages(component, configurations)()
                }
              }
            } else {
              log.info("Pipeline", "No components changed, skipping...")
            }
          }
        }
      }

    }
  }
}