from rest_framework import viewsets from .models import SourceNetwork from .serializers import NetworkSourceSerializer from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated from datetime import datetime from rest_framework.decorators import action from rest_framework.response import Response class NetworkSourceViewSet(viewsets.ModelViewSet): queryset = SourceNetwork.objects.all() serializer_class = NetworkSourceSerializer @action(detail=True, methods=['post']) def run_scraper(self, request, pk=None): # Możesz tutaj odpalić Celery task lub inny backendowy job source = self.get_object() # Example: run_network_scraper.delay(source.id) # <- Celery task # albo odpalić subprocess (jeśli synchronizujesz) # subprocess.Popen(["python", "manage.py", "run_single_scraper", str(source.id)]) return Response({"status": f"Scraper for {source.title} triggered."}) @api_view(['POST']) @permission_classes([IsAuthenticated]) def start_scraper(request): """ POST body: { "source_id": int, "source_name": str, ... } """ source_id = request.data.get('source_id') source_name = request.data.get('source_name') params = request.data.get('params', {}) # Jeśli podano name, znajdź id: if source_name: from extractly.models import SourceNetwork try: source = SourceNetwork.objects.get(title=source_name) source_id = source.id except SourceNetwork.DoesNotExist: return Response({"error": f"Nie znaleziono źródła o nazwie '{source_name}'"}, status=404) # dalej normalnie... celery_kwargs = {} if source_id: celery_kwargs['source_id'] = int(source_id) if params: celery_kwargs['params'] = params # ... reszta jak masz! import redis import json from django.conf import settings from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response @api_view(['GET']) @permission_classes([IsAuthenticated]) def celery_queue(request): """ Wyświetla listę zadań oczekujących w kolejce Celery (pending). """ # Połącz się z brokerem Redis redis_url = settings.CELERY_BROKER_URL.replace("redis://", "") # Uwaga: to nie jest superbezpieczne dla środowisk produkcyjnych (lepsza konfiguracja = lepsze security!) host_port_db = redis_url.split("/") host, port = host_port_db[0].split(":") db = int(host_port_db[1]) if len(host_port_db) > 1 else 0 r = redis.Redis(host=host, port=port, db=db) # Celery domyślnie używa listy "celery" na brokera redis queue_name = "celery" length = r.llen(queue_name) tasks = [] for i in range(length): try: item = r.lindex(queue_name, i) if item: # Celery używa AMQP, serializacja JSON (można podejrzeć taska) data = json.loads(item) tasks.append({ "name": data.get("headers", {}).get("task"), "id": data.get("headers", {}).get("id"), "args": data.get("body"), # Dodaj tu więcej pól jeśli chcesz }) except Exception as e: tasks.append({"error": str(e)}) return Response({ "pending_tasks": tasks, "total": length }) from django_celery_results.models import TaskResult @api_view(['GET']) @permission_classes([IsAuthenticated]) def celery_task_history(request): """ Lista wszystkich tasków z bazy (celery-results). """ tasks = TaskResult.objects.order_by('-date_done')[:50] # ostatnie 50 return Response([ { "task_id": t.task_id, "task_name": t.task_name, "status": t.status, "date_done": t.date_done, "result": t.result, } for t in tasks ]) from celery import current_app @api_view(['GET']) @permission_classes([IsAuthenticated]) def celery_active(request): """ Lista aktywnych/wykonywanych tasków. """ i = current_app.control.inspect() active = i.active() return Response(active) @api_view(['GET']) @permission_classes([IsAuthenticated]) def celery_status(request): # TODO: pobierz kolejkę (Redis), historię (celery-results), aktywne (inspect) # ...zwróć w jednej odpowiedzi! return Response({ "queue": [...], "history": [...], "active": [...], }) from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule import json @api_view(['POST']) @permission_classes([IsAuthenticated]) def create_periodic_task(request): """ Tworzy nowy cykliczny task Celery (PeriodicTask). POST body: { "name": "Playwright run o 2 w nocy", "task": "extractly.tasks.run_playwright_scraper", "interval": {"every": 1, "period": "hours"}, # LUB "crontab": {"minute": "0", "hour": "2", "day_of_week": "*", "day_of_month": "*", "month_of_year": "*"}, "args": [5], "kwargs": {"source_id": 5, "params": {...}}, "enabled": true } """ data = request.data # INTERVAL lub CRONTAB schedule = None if "interval" in data: interval, created = IntervalSchedule.objects.get_or_create( every=data["interval"]["every"], period=data["interval"]["period"] ) schedule_type = "interval" schedule = interval elif "crontab" in data: crontab, created = CrontabSchedule.objects.get_or_create( minute=data["crontab"].get("minute", "*"), hour=data["crontab"].get("hour", "*"), day_of_week=data["crontab"].get("day_of_week", "*"), day_of_month=data["crontab"].get("day_of_month", "*"), month_of_year=data["crontab"].get("month_of_year", "*"), timezone="UTC" ) schedule_type = "crontab" schedule = crontab else: return Response({"error": "Wymagany schedule: interval lub crontab"}, status=400) args = json.dumps(data.get("args", [])) kwargs = json.dumps(data.get("kwargs", {})) name = data.get("name") task = data.get("task") enabled = data.get("enabled", True) periodic_task, created = PeriodicTask.objects.update_or_create( name=name, defaults={ "task": task, "interval": schedule if schedule_type == "interval" else None, "crontab": schedule if schedule_type == "crontab" else None, "args": args, "kwargs": kwargs, "enabled": enabled, } ) return Response({ "id": periodic_task.id, "name": periodic_task.name, "enabled": periodic_task.enabled, "schedule": data.get("interval") or data.get("crontab"), "args": args, "kwargs": kwargs, "created": created, }) @api_view(['GET']) @permission_classes([IsAuthenticated]) def list_periodic_tasks(request): tasks = PeriodicTask.objects.all().order_by('-id') out = [] for t in tasks: out.append({ "id": t.id, "name": t.name, "task": t.task, "enabled": t.enabled, "interval": str(t.interval) if t.interval else None, "crontab": str(t.crontab) if t.crontab else None, "args": t.args, "kwargs": t.kwargs, }) return Response(out) @api_view(['GET']) @permission_classes([IsAuthenticated]) def get_periodic_task(request, pk): try: t = PeriodicTask.objects.get(pk=pk) except PeriodicTask.DoesNotExist: return Response({'error': 'Task nie istnieje'}, status=404) return Response({ "id": t.id, "name": t.name, "task": t.task, "enabled": t.enabled, "interval": str(t.interval) if t.interval else None, "crontab": str(t.crontab) if t.crontab else None, "args": t.args, "kwargs": t.kwargs, }) @api_view(['PATCH']) @permission_classes([IsAuthenticated]) def update_periodic_task(request, pk): try: t = PeriodicTask.objects.get(pk=pk) except PeriodicTask.DoesNotExist: return Response({'error': 'Task nie istnieje'}, status=404) data = request.data if 'name' in data: t.name = data['name'] if 'task' in data: t.task = data['task'] if 'enabled' in data: t.enabled = data['enabled'] if 'args' in data: t.args = json.dumps(data['args']) if 'kwargs' in data: t.kwargs = json.dumps(data['kwargs']) # harmonogram if 'interval' in data: interval_data = data['interval'] interval, _ = IntervalSchedule.objects.get_or_create( every=interval_data['every'], period=interval_data['period'], ) t.interval = interval t.crontab = None if 'crontab' in data: crontab_data = data['crontab'] crontab, _ = CrontabSchedule.objects.get_or_create( minute=crontab_data.get('minute', '*'), hour=crontab_data.get('hour', '*'), day_of_week=crontab_data.get('day_of_week', '*'), day_of_month=crontab_data.get('day_of_month', '*'), month_of_year=crontab_data.get('month_of_year', '*'), timezone=crontab_data.get('timezone', 'UTC') ) t.crontab = crontab t.interval = None t.save() return Response({'status': 'updated'}) @api_view(['DELETE']) @permission_classes([IsAuthenticated]) def delete_periodic_task(request, pk): try: t = PeriodicTask.objects.get(pk=pk) t.delete() return Response({'deleted': pk}) except PeriodicTask.DoesNotExist: return Response({'error': 'Task nie istnieje'}, status=404) @api_view(['POST']) @permission_classes([IsAuthenticated]) def toggle_periodic_task(request, pk): try: t = PeriodicTask.objects.get(pk=pk) t.enabled = not t.enabled t.save() return Response({'id': pk, 'enabled': t.enabled}) except PeriodicTask.DoesNotExist: return Response({'error': 'Task nie istnieje'}, status=404) # manual_agregator/api/views.py from rest_framework import viewsets, permissions, filters from extractly.models import ( SourceHtml, # albo NetworkSource jeśli to ta klasa SourceManual, NetworkMonitoredPage, AdsManual, Ads, # doprecyzuj jeśli potrzebujesz inny model/statystyki ) from extractly.serializers import ( HtmlNetworkSourceSerializer, ManualDataSourceSerializer, NetworkMonitoredPageSerializer, NetworkMonitoringManualSerializer, NetworkMonitoringSerializer, ) from extractly.filters import ( HtmlNetworkSourceFilter, ManualDataSourceFilter, NetworkMonitoredPageFilter, NetworkMonitoringManualFilter, NetworkMonitoringFilter, ) from extractly.pagination import StandardResultsSetPagination class HtmlNetworkSourceViewSet(viewsets.ModelViewSet): queryset = SourceHtml.objects.all() serializer_class = HtmlNetworkSourceSerializer filterset_class = HtmlNetworkSourceFilter permission_classes = [permissions.AllowAny] pagination_class = StandardResultsSetPagination class ManualDataSourceViewSet(viewsets.ModelViewSet): queryset = SourceManual.objects.all() serializer_class = ManualDataSourceSerializer filterset_class = ManualDataSourceFilter permission_classes = [permissions.AllowAny] pagination_class = StandardResultsSetPagination class NetworkMonitoredPageViewSet(viewsets.ModelViewSet): queryset = NetworkMonitoredPage.objects.all() serializer_class = NetworkMonitoredPageSerializer filterset_class = NetworkMonitoredPageFilter permission_classes = [permissions.IsAuthenticated] pagination_class = StandardResultsSetPagination class NetworkMonitoringManualViewSet(viewsets.ModelViewSet): queryset = AdsManual.objects.all() serializer_class = NetworkMonitoringManualSerializer filterset_class = NetworkMonitoringManualFilter permission_classes = [permissions.IsAuthenticated] pagination_class = StandardResultsSetPagination class NetworkMonitoringViewSet(viewsets.ModelViewSet): queryset = Ads.objects.all() serializer_class = NetworkMonitoringSerializer filterset_class = NetworkMonitoringFilter permission_classes = [permissions.IsAuthenticated] pagination_class = StandardResultsSetPagination