const express = require('express')
const cheerio = require('cheerio')
const puppeteer = require('puppeteer-extra')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')

puppeteer.use(StealthPlugin())

const app = express()
app.use(express.json())

const CHROME_PATH = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium'
const PORT = process.env.PORT || 3001
const BYPARR_URL = process.env.BYPARR_URL || ''

// ---------- Byparr helpers ----------

async function byparrRequest(body) {
  const resp = await fetch(`${BYPARR_URL}/v1`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(body),
  })
  if (!resp.ok) throw new Error(`byparr HTTP ${resp.status}: ${await resp.text()}`)
  return resp.json()
}

async function byparrFetch(url) {
  console.log('[byparr] GET', url)
  const data = await byparrRequest({ cmd: 'request.get', url, maxTimeout: 60000 })
  const html = data.solution?.response
  if (!html) throw new Error(`byparr returned no HTML for ${url}`)
  return { html }
}

// ---------- Article parsing ----------

function parseArticlesFromHtml(html) {
  const $ = cheerio.load(html)
  const results = []
  const seen = new Set()

  $('a[data-component="story-link"]').each((_, el) => {
    const a = $(el)
    const href = a.attr('href') || ''
    const absolute = href.startsWith('http') ? href : `https://www.bloomberg.com${href}`
    const clean = absolute.split('?')[0]

    if (seen.has(clean)) return
    if (!/\/(news\/articles|news\/features|features)\//.test(clean)) return

    const title = a.find('[data-testid="headline"] span').text().trim()
    if (!title || title.length < 10) return

    seen.add(clean)
    results.push({ title, url: clean })
  })

  return results
}

// ---------- Bloomberg scraping via Byparr ----------

async function scrapeBloombergViaByparr(username, password) {
  // /markets main page is excluded: live ticker widgets cause continuous network activity
  // so Byparr's networkidle wait never resolves. Sub-sections below are editorial pages
  // without live data — they behave like /technology and /economics (~5s each).
  const byparrPages = [
    'https://www.bloomberg.com/markets/stocks',
    'https://www.bloomberg.com/markets/currencies',
    'https://www.bloomberg.com/markets/rates-bonds',
    'https://www.bloomberg.com/markets/commodities',
    'https://www.bloomberg.com/technology',
    'https://www.bloomberg.com/economics',
    'https://www.bloomberg.com/politics',
  ]

  const articles = []
  const seen = new Set()

  for (const url of byparrPages) {
    try {
      const { html } = await byparrFetch(url)
      const items = parseArticlesFromHtml(html)
      for (const item of items) {
        if (!seen.has(item.url)) {
          seen.add(item.url)
          articles.push(item)
        }
      }
      console.log('[bloomberg]', url, '->', items.length, 'articles')
    } catch (e) {
      console.error('[bloomberg] error on', url, ':', e.message)
    }
  }

  return articles
}

// ---------- Bloomberg scraping via Puppeteer (fallback, may be blocked) ----------

function launchBrowser() {
  return puppeteer.launch({
    executablePath: CHROME_PATH,
    headless: true,
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage',
      '--disable-gpu',
      '--window-size=1920,1080',
      '--disable-blink-features=AutomationControlled',
    ],
  })
}

async function tryClick(page, selectors) {
  for (const sel of selectors) {
    try {
      const el = await page.$(sel)
      if (el) { await el.click(); return true }
    } catch {}
  }
  await page.keyboard.press('Enter')
  return false
}

async function tryType(page, selectors, text) {
  for (const sel of selectors) {
    try {
      await page.waitForSelector(sel, { timeout: 4000 })
      await page.type(sel, text, { delay: 60 })
      return true
    } catch {}
  }
  return false
}

async function scrapeBloombergViaPuppeteer(username, password) {
  let browser
  try {
    browser = await launchBrowser()
    const page = await browser.newPage()
    await page.setViewport({ width: 1920, height: 1080 })

    await page.evaluateOnNewDocument(() => {
      Object.defineProperty(navigator, 'webdriver', { get: () => undefined })
      window.chrome = { runtime: {} }
      Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] })
    })

    console.log('[bloomberg] navigating to login page')
    await page.goto('https://www.bloomberg.com/account/signin', {
      waitUntil: 'networkidle2',
      timeout: 60000,
    })
    await new Promise(r => setTimeout(r, 2000))

    const pageInputs = await page.evaluate(() =>
      Array.from(document.querySelectorAll('input')).map(i => ({
        type: i.type, name: i.name, id: i.id, placeholder: i.placeholder, visible: i.offsetParent !== null
      }))
    )
    console.log('[bloomberg] inputs on login page:', JSON.stringify(pageInputs))
    console.log('[bloomberg] page title:', await page.title())

    const emailOk = await tryType(page, [
      '#email-form-input',
      'input[id="email-form-input"]',
      'input[type="email"]',
      'input[name="text-input"]',
      'input[placeholder*="email" i]',
    ], username)
    if (!emailOk) throw new Error('could not find email input')

    await new Promise(r => setTimeout(r, 800))

    const submitted = await page.evaluate(() => {
      const btn = Array.from(document.querySelectorAll('button')).find(b =>
        b.type === 'submit' || /continue|next|sign.?in/i.test(b.textContent)
      )
      if (btn) { btn.click(); return true }
      const form = document.querySelector('form')
      if (form) { form.submit(); return true }
      return false
    })
    if (!submitted) await page.keyboard.press('Enter')

    try {
      await page.waitForFunction(
        () => document.querySelector('input[type="password"]') !== null,
        { timeout: 10000 }
      )
    } catch {
      await new Promise(r => setTimeout(r, 3000))
    }
    console.log('[bloomberg] after email submit, url:', page.url())

    const pwdOk = await tryType(page, [
      'input[type="password"]',
      'input[name="password"]',
      'input[autocomplete="current-password"]',
      'input[autocomplete="password"]',
    ], password)
    if (!pwdOk) throw new Error('could not find password input')

    await new Promise(r => setTimeout(r, 500))
    await Promise.all([
      page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 20000 }).catch(() => {}),
      tryClick(page, ['button[type="submit"]', 'input[type="submit"]']),
    ])

    const currentURL = page.url()
    console.log('[bloomberg] after login, url:', currentURL)
    if (currentURL.includes('/account/signin')) {
      const bodyText = await page.evaluate(() => document.body.innerText.slice(0, 500))
      throw new Error(`login did not redirect — page content: ${bodyText}`)
    }

    const targetPages = [
      'https://www.bloomberg.com/markets',
      'https://www.bloomberg.com/technology',
      'https://www.bloomberg.com/economics',
    ]

    const articles = []
    const seen = new Set()

    for (const url of targetPages) {
      try {
        console.log('[bloomberg] scraping', url)
        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 })
        await page.waitForSelector('a[data-component="story-link"]', { timeout: 15000 }).catch(() => {})

        const html = await page.content()
        const items = parseArticlesFromHtml(html)

        for (const item of items) {
          if (!seen.has(item.url)) {
            seen.add(item.url)
            articles.push(item)
          }
        }
        console.log('[bloomberg]', url, '->', items.length, 'articles')
      } catch (e) {
        console.error('[bloomberg] error on', url, ':', e.message)
      }
    }

    return articles
  } finally {
    if (browser) await browser.close()
  }
}

// ---------- Routes ----------

app.get('/health', (_, res) => res.json({ ok: true }))

app.post('/bloomberg/scrape', async (req, res) => {
  const { username, password } = req.body || {}
  if (!username || !password) {
    return res.status(400).json({ error: 'username and password required' })
  }

  try {
    let articles
    if (BYPARR_URL) {
      console.log('[bloomberg] using Byparr (nodriver) to bypass bot detection')
      articles = await scrapeBloombergViaByparr(username, password)
    } else {
      console.log('[bloomberg] using Puppeteer (Byparr not configured)')
      articles = await scrapeBloombergViaPuppeteer(username, password)
    }

    console.log('[bloomberg] total:', articles.length, 'articles')
    res.json({ articles })
  } catch (e) {
    console.error('[bloomberg] scrape error:', e.message)
    res.status(500).json({ error: e.message })
  }
})

app.listen(PORT, () => console.log(`scraper-service listening on :${PORT}`))