-- =========================================================================
-- SQL Script to INSERT Morizon SourceHtml Configuration
-- =========================================================================
-- Table: extractly_sourcehtml
-- Purpose: Configure Level 2 (HTML Collection) for Morizon
-- =========================================================================

-- IMPORTANT: Replace the JSON content in actions/selectors/inactive/raw_data_setup
-- with the actual content from the respective JSON files (WITHOUT _comment/_usage fields)

INSERT INTO extractly_sourcehtml (
    id,
    type,
    title,
    name,
    actions,
    selectors,
    inactive,
    raw_data_setup,
    source_id
) VALUES (
    -- Use next available ID or specific ID you want
    (SELECT COALESCE(MAX(id), 0) + 1 FROM extractly_sourcehtml),
    
    -- Type
    'morizon html',
    
    -- Title
    'HTML collector for Morizon real estate listings',
    
    -- Name (must match source name from SourceNetwork for linking)
    'morizon',
    
    -- Actions: Copy ONLY the array from morizon_actions.json (remove wrapper object)
    -- Start from line 16, the [ bracket, until the closing ]
    '[
        {
            "name": "accept_cookies_if_present",
            "type": "click_if_exists",
            "timeout": 45000,
            "selectors": [
                "#rasp_cmp .cmp-intro_acceptAll",
                "button.cmp-intro_acceptAll",
                "button[aria-label=''Przejdź do serwisu'']",
                "#rasp_cmp [tabindex=''0''][aria-label=''Przejdź do serwisu'']",
                "button.sv__btn-close",
                "button[aria-label=''Close the survey'']"
            ],
            "pre_wait_for": [
                "#rasp_cmp",
                ".cmp-popup_popup.show_amp",
                ".cmp-popup_content.cmp-popup_firstView"
            ],
            "post_wait_for_detach": "#rasp_cmp"
        },
        {
            "name": "fallback_escape",
            "type": "press_if_visible",
            "key": "Escape",
            "timeout": 4000,
            "selector": "#rasp_cmp"
        },
        {
            "name": "close_survey_popup",
            "type": "click_if_exists",
            "timeout": 5000,
            "selectors": [
                "button.sv__btn-close",
                "button[aria-label=''Close the survey'']",
                "[data-testid=''survey''] button[aria-label*=''Close'']"
            ]
        },
        {
            "name": "wait_main_content",
            "type": "wait_for_selector_any",
            "timeout": 60000,
            "selectors": [
                "main",
                "article",
                ".content",
                "[data-cy=''adPageContent'']",
                "[data-testid=''ad-page'']"
            ]
        },
        {
            "name": "expand_description",
            "type": "click_text",
            "text": "Zobacz opis",
            "index": 0,
            "timeout": 5000,
            "optional": true
        },
        {
            "name": "expand_description_alt",
            "type": "click_text",
            "text": "Pokaż więcej",
            "index": 0,
            "timeout": 5000,
            "optional": true
        },
        {
            "name": "show_phone_number",
            "type": "click_text",
            "text": "Pokaż numer",
            "index": 0,
            "timeout": 5000,
            "optional": true
        }
    ]'::jsonb,
    
    -- Selectors: Copy from morizon_selectors.json (remove _comment/_usage/_instructions)
    '{
        "list": {
            "pagination": {
                "strategy": "url_change",
                "total_pages_css": "link[rel=''next''], nav[aria-label*=''Pag''] a, .pagination a, ul.pagination li a",
                "page_url_template": "?page={{number}}"
            },
            "item_link_css": "a[href^=''/oferta/''], article a[href^=''/oferta/''], a[data-cy=''propertyUrl'']",
            "item_link_attr": "href"
        },
        "detail": {
            "fields": {
                "title": ["$.name", "meta[property=''og:title'']@content", "title", "h1"],
                "price": ["$.offers.price", "$.price"],
                "currency": ["$.offers.priceCurrency", "$.priceCurrency"],
                "description": ["$.description", "meta[name=''description'']@content", "meta[property=''og:description'']@content"],
                "square_footage": ["$.itemOffered.floorSize.value", "$.floorSize.value"],
                "rooms": ["$.itemOffered.numberOfRooms", "$.numberOfRooms"],
                "floor": ["$.itemOffered.floorLevel", "$.floorLevel"],
                "address.street": ["$.itemOffered.address.streetAddress", "$.address.streetAddress"],
                "address.city": ["$.itemOffered.address.addressLocality", "$.address.addressLocality"],
                "address.region": ["$.itemOffered.address.addressRegion", "$.address.addressRegion"],
                "images[]": ["$.image", "meta[property=''og:image'']@content"]
            },
            "canonical": "link[rel=''canonical'']@href",
            "jsonld_css": "script[type=''application/ld+json'']",
            "jsonld_pick_types": ["Offer", "Product", "RealEstateListing", "Accommodation", "SingleFamilyResidence", "Apartment"],
            "fallback_regex": {
                "price": "(\\d[\\d\\s.,]+)\\s?(PLN|zł)",
                "address.city": "(?i)\\b([A-ZŁŚŻŹĆŃÓĘ][\\p{L}]+)\\b(?:,|\\s)",
                "address.street": "(?i)ul\\.\\s*([^,]+)",
                "square_footage": "([\\d.,]+)\\s*m²"
            }
        },
        "sliced_html": {
            "merge": true,
            "content_type": "41",
            "content_field": "sliced_html",
            "skip_selectors": ["#rasp_cmp", "script", "noscript", "style", "header", "footer", "nav", "[aria-hidden=''true'']", "[class*=''cookie'']", "[class*=''Cookie'']", "[class*=''banner'']", "[class*=''Banner'']", "[data-testid*=''cookie'']", "[data-testid*=''survey'']", ".sv__survey", ".newsletter", ".ads", "[id*=''ad-'']", "[class*=''advertisement'']", "[class*=''sidebar'']"],
            "sliced_gallery_html": {
                "selector": "div[class*=''gallery''], div[class*=''Gallery''], section[class*=''Gallery''], [data-cy*=''gallery''], [class*=''Carousel''], div.gallery-slider, [data-testid*=''gallery'']",
                "playwright_action": "inner_html"
            },
            "sliced_container_html": {
                "selector": "main, article, .content, [data-cy=''adPageContent''], [data-testid*=''ad-page''], [data-testid=''ad-details''], .listing-content, .offer-details, .property-details",
                "playwright_action": "inner_html"
            },
            "sliced_details_html": {
                "selector": "[data-cy=''propertyCardInfo''], .property-info, .card-offer-info, div[class*=''details''], section[class*=''Details'']",
                "playwright_action": "inner_html"
            }
        }
    }'::jsonb,
    
    -- Inactive: Copy from morizon_inactive.json (remove _comment/_usage/_instructions)
    '{
        "css_exists_any": [
            "[data-testid*=''archived'']",
            "[data-cy*=''archived'']",
            "div[class*=''archiw'']",
            "div[class*=''Archiw'']",
            "div[class*=''nieaktualn'']",
            "div[class*=''Nieaktualn'']",
            "div[class*=''expired'']",
            "div[class*=''Expired'']",
            ".inactive",
            ".archived"
        ],
        "http_status_in": [404, 410, 451, 500],
        "text_contains_any": [
            "ogłoszenie wygasło",
            "oferta nieaktualna",
            "archiwalne",
            "oferta została usunięta",
            "ogłoszenie nie istnieje",
            "strona nie została znaleziona",
            "404",
            "nie znaleziono"
        ]
    }'::jsonb,
    
    -- Raw Data Setup: Copy from morizon_raw_data_setup.json (remove _comment/_usage/_instructions)
    '{
        "meta_dump": {
            "og_title": "meta[property=''og:title'']@content",
            "og_image": "meta[property=''og:image'']@content",
            "og_url": "meta[property=''og:url'']@content",
            "og_type": "meta[property=''og:type'']@content",
            "og_site_name": "meta[property=''og:site_name'']@content",
            "description": "meta[name=''description'']@content",
            "og_description": "meta[property=''og:description'']@content",
            "canonical": "link[rel=''canonical'']@href",
            "keywords": "meta[name=''keywords'']@content",
            "author": "meta[name=''author'']@content"
        },
        "jsonld_dump": {
            "css": "script[type=''application/ld+json'']",
            "store_as": "jsonld_all",
            "parse_json": true
        },
        "additional_metadata": {
            "page_title": "title",
            "main_heading": "h1",
            "property_id": "[data-cy=''propertyId''], [data-testid=''propertyId''], .property-id, .listing-id"
        }
    }'::jsonb,
    
    -- Source ID: Morizon UUID from extractly_sourcenetwork
    '779dc8fa-ef0b-4627-8cf6-755f3375d949'
);

-- =========================================================================
-- Verification Query
-- =========================================================================

SELECT 
    id,
    name,
    type,
    title,
    source_id,
    jsonb_array_length(actions) as actions_count,
    jsonb_typeof(selectors) as selectors_type,
    jsonb_typeof(inactive) as inactive_type,
    jsonb_typeof(raw_data_setup) as raw_data_type
FROM extractly_sourcehtml
WHERE source_id = '779dc8fa-ef0b-4627-8cf6-755f3375d949';

-- =========================================================================
-- Alternative: UPDATE if row already exists
-- =========================================================================

-- If you already have a row and just want to update it:
/*
UPDATE extractly_sourcehtml
SET
    type = 'morizon html',
    title = 'HTML collector for Morizon real estate listings',
    name = 'morizon',
    actions = '[ ... paste actions JSON ... ]'::jsonb,
    selectors = '{ ... paste selectors JSON ... }'::jsonb,
    inactive = '{ ... paste inactive JSON ... }'::jsonb,
    raw_data_setup = '{ ... paste raw_data_setup JSON ... }'::jsonb
WHERE source_id = '779dc8fa-ef0b-4627-8cf6-755f3375d949';
*/
