277 lines
10 KiB
Python
277 lines
10 KiB
Python
"""
|
|
Webhook processing service
|
|
Implements intelligent dispatch, task queueing, and deduplication strategy
|
|
"""
|
|
|
|
import asyncio
|
|
from typing import Optional, Dict, Any
|
|
from datetime import datetime
|
|
import structlog
|
|
from celery import Celery
|
|
|
|
from app.config import get_settings
|
|
from app.models.gitea import GiteaWebhook, WebhookResponse
|
|
from app.services.dedup_service import DeduplicationService
|
|
from app.services.jenkins_service import JenkinsService
|
|
from app.services.database_service import get_database_service
|
|
from app.tasks.jenkins_tasks import trigger_jenkins_job
|
|
|
|
logger = structlog.get_logger()
|
|
|
|
class WebhookService:
|
|
"""Webhook processing service"""
|
|
|
|
def __init__(
|
|
self,
|
|
dedup_service: DeduplicationService,
|
|
jenkins_service: JenkinsService,
|
|
celery_app: Celery
|
|
):
|
|
self.dedup_service = dedup_service
|
|
self.jenkins_service = jenkins_service
|
|
self.celery_app = celery_app
|
|
self.settings = get_settings()
|
|
self.db_service = get_database_service()
|
|
|
|
async def process_webhook(self, webhook: GiteaWebhook) -> WebhookResponse:
|
|
"""
|
|
Process webhook event
|
|
Args:
|
|
webhook: Gitea webhook data
|
|
Returns:
|
|
WebhookResponse: processing result
|
|
"""
|
|
try:
|
|
# 1. Validate event type
|
|
if not webhook.is_push_event():
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Non-push event ignored",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 2. Extract key information
|
|
branch = webhook.get_branch_name()
|
|
commit_hash = webhook.get_commit_hash()
|
|
repository = webhook.repository.full_name
|
|
|
|
logger.info("Processing webhook",
|
|
repository=repository,
|
|
branch=branch,
|
|
commit_hash=commit_hash)
|
|
|
|
# 3. Deduplication check
|
|
dedup_key = self.dedup_service.generate_dedup_key(commit_hash, branch)
|
|
if await self.dedup_service.is_duplicate(dedup_key):
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Duplicate event ignored",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 4. Get project mapping and job name
|
|
job_name = await self._determine_job_name(repository, branch)
|
|
if not job_name:
|
|
return WebhookResponse(
|
|
success=True,
|
|
message=f"No Jenkins job mapping for repository: {repository}, branch: {branch}",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
# 5. Prepare job parameters
|
|
job_params = self._prepare_job_parameters(webhook, job_name)
|
|
|
|
# 6. Submit job to queue
|
|
task_result = await self._submit_job_to_queue(
|
|
webhook, job_name, job_params
|
|
)
|
|
|
|
if task_result:
|
|
return WebhookResponse(
|
|
success=True,
|
|
message="Job queued successfully",
|
|
event_id=webhook.get_event_id(),
|
|
job_name=job_name
|
|
)
|
|
else:
|
|
return WebhookResponse(
|
|
success=False,
|
|
message="Failed to queue job",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error("Error processing webhook",
|
|
repository=webhook.repository.full_name,
|
|
error=str(e))
|
|
return WebhookResponse(
|
|
success=False,
|
|
message=f"Internal server error: {str(e)}",
|
|
event_id=webhook.get_event_id()
|
|
)
|
|
|
|
async def _determine_job_name(self, repository: str, branch: str) -> Optional[str]:
|
|
"""Determine job name by repository and branch"""
|
|
# First try to get project mapping from database
|
|
job_name = await self.db_service.determine_job_name(repository, branch)
|
|
if job_name:
|
|
return job_name
|
|
|
|
# If not found in database, use environment dispatch from config
|
|
environment = self.settings.get_environment_for_branch(branch)
|
|
if environment:
|
|
return environment.jenkins_job
|
|
|
|
return None
|
|
|
|
def _prepare_job_parameters(self, webhook: GiteaWebhook, job_name: str) -> Dict[str, str]:
|
|
"""Prepare Jenkins job parameters"""
|
|
author_info = webhook.get_author_info()
|
|
|
|
return {
|
|
"BRANCH_NAME": webhook.get_branch_name(),
|
|
"COMMIT_SHA": webhook.get_commit_hash(),
|
|
"REPOSITORY_URL": webhook.repository.clone_url,
|
|
"REPOSITORY_NAME": webhook.repository.full_name,
|
|
"PUSHER_NAME": author_info["name"],
|
|
"PUSHER_EMAIL": author_info["email"],
|
|
"PUSHER_USERNAME": author_info["username"],
|
|
"COMMIT_MESSAGE": webhook.get_commit_message(),
|
|
"JOB_NAME": job_name,
|
|
"WEBHOOK_EVENT_ID": webhook.get_event_id(),
|
|
"TRIGGER_TIME": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
async def _submit_job_to_queue(
|
|
self,
|
|
webhook: GiteaWebhook,
|
|
job_name: str,
|
|
job_params: Dict[str, str]
|
|
) -> bool:
|
|
"""Submit job to Celery queue"""
|
|
try:
|
|
# Create task
|
|
task_kwargs = {
|
|
"job_name": job_name,
|
|
"jenkins_url": self.settings.jenkins.url,
|
|
"parameters": job_params,
|
|
"event_id": webhook.get_event_id(),
|
|
"repository": webhook.repository.full_name,
|
|
"branch": webhook.get_branch_name(),
|
|
"commit_hash": webhook.get_commit_hash(),
|
|
"priority": 1 # Default priority
|
|
}
|
|
|
|
# Submit to Celery queue
|
|
task = self.celery_app.send_task(
|
|
"app.tasks.jenkins_tasks.trigger_jenkins_job",
|
|
kwargs=task_kwargs,
|
|
priority=task_kwargs["priority"]
|
|
)
|
|
|
|
logger.info("Job submitted to queue",
|
|
task_id=task.id,
|
|
job_name=job_name,
|
|
repository=webhook.repository.full_name,
|
|
branch=webhook.get_branch_name())
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to submit job to queue",
|
|
job_name=job_name,
|
|
error=str(e))
|
|
return False
|
|
|
|
async def get_webhook_stats(self) -> Dict[str, Any]:
|
|
"""Get webhook processing statistics"""
|
|
try:
|
|
# Get queue stats
|
|
queue_stats = await self._get_queue_stats()
|
|
|
|
# Get deduplication stats
|
|
dedup_stats = await self.dedup_service.get_stats()
|
|
|
|
# Get environment config
|
|
environments = {}
|
|
for name, config in self.settings.environments.items():
|
|
environments[name] = {
|
|
"branches": config.branches,
|
|
"jenkins_job": config.jenkins_job,
|
|
"jenkins_url": config.jenkins_url,
|
|
"priority": config.priority
|
|
}
|
|
|
|
return {
|
|
"queue": queue_stats,
|
|
"deduplication": dedup_stats,
|
|
"environments": environments,
|
|
"config": {
|
|
"max_concurrent": self.settings.queue.max_concurrent,
|
|
"max_retries": self.settings.queue.max_retries,
|
|
"retry_delay": self.settings.queue.retry_delay
|
|
},
|
|
"timestamp": datetime.utcnow().isoformat()
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error getting webhook stats", error=str(e))
|
|
return {"error": str(e)}
|
|
|
|
async def _get_queue_stats(self) -> Dict[str, Any]:
|
|
"""Get queue statistics"""
|
|
try:
|
|
# Get Celery queue stats
|
|
inspect = self.celery_app.control.inspect()
|
|
|
|
# Active tasks
|
|
active = inspect.active()
|
|
active_count = sum(len(tasks) for tasks in active.values()) if active else 0
|
|
|
|
# Reserved tasks
|
|
reserved = inspect.reserved()
|
|
reserved_count = sum(len(tasks) for tasks in reserved.values()) if reserved else 0
|
|
|
|
# Registered workers
|
|
registered = inspect.registered()
|
|
worker_count = len(registered) if registered else 0
|
|
|
|
return {
|
|
"active_tasks": active_count,
|
|
"queued_tasks": reserved_count,
|
|
"worker_count": worker_count,
|
|
"queue_length": active_count + reserved_count
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error getting queue stats", error=str(e))
|
|
return {"error": str(e)}
|
|
|
|
async def clear_queue(self) -> Dict[str, Any]:
|
|
"""Clear queue"""
|
|
try:
|
|
# Revoke all active tasks
|
|
inspect = self.celery_app.control.inspect()
|
|
active = inspect.active()
|
|
|
|
revoked_count = 0
|
|
if active:
|
|
for worker, tasks in active.items():
|
|
for task in tasks:
|
|
self.celery_app.control.revoke(task["id"], terminate=True)
|
|
revoked_count += 1
|
|
|
|
logger.info("Queue cleared", revoked_count=revoked_count)
|
|
|
|
return {
|
|
"success": True,
|
|
"revoked_count": revoked_count,
|
|
"message": f"Cleared {revoked_count} tasks from queue"
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Error clearing queue", error=str(e))
|
|
return {
|
|
"success": False,
|
|
"error": str(e)
|
|
} |