feat: add frontend + backend + database to retrieve and compute news from Yahoo

backend/internal/scraper/stocktwits/stocktwits.go

@@ -0,0 +1,128 @@
+package stocktwits
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/tradarr/backend/internal/scraper"
+)
+
+const apiBase = "https://api.stocktwits.com/api/2"
+
+type StockTwits struct {
+	client *http.Client
+}
+
+func New() *StockTwits {
+	return &StockTwits{
+		client: &http.Client{Timeout: 15 * time.Second},
+	}
+}
+
+func (s *StockTwits) Name() string { return "stocktwits" }
+
+type apiResponse struct {
+	Response struct {
+		Status int    `json:"status"`
+		Error  string `json:"error,omitempty"`
+	} `json:"response"`
+	Messages []struct {
+		ID        int    `json:"id"`
+		Body      string `json:"body"`
+		CreatedAt string `json:"created_at"`
+		User      struct {
+			Username string `json:"username"`
+		} `json:"user"`
+		Entities struct {
+			Sentiment *struct {
+				Basic string `json:"basic"`
+			} `json:"sentiment"`
+		} `json:"entities"`
+	} `json:"messages"`
+}
+
+func (s *StockTwits) Scrape(ctx context.Context, symbols []string) ([]scraper.Article, error) {
+	var articles []scraper.Article
+	for i, symbol := range symbols {
+		// Délai entre les requêtes pour éviter le rate limiting
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return articles, ctx.Err()
+			case <-time.After(500 * time.Millisecond):
+			}
+		}
+		msgs, err := s.fetchSymbol(ctx, symbol)
+		if err != nil {
+			fmt.Printf("stocktwits %s: %v\n", symbol, err)
+			continue
+		}
+		articles = append(articles, msgs...)
+	}
+	return articles, nil
+}
+
+func (s *StockTwits) fetchSymbol(ctx context.Context, symbol string) ([]scraper.Article, error) {
+	url := fmt.Sprintf("%s/streams/symbol/%s.json", apiBase, symbol)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36")
+
+	resp, err := s.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode == 429 {
+		return nil, fmt.Errorf("rate limited by StockTwits for %s", symbol)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("StockTwits returned HTTP %d for %s: %s", resp.StatusCode, symbol, string(body))
+	}
+
+	var data apiResponse
+	if err := json.Unmarshal(body, &data); err != nil {
+		return nil, fmt.Errorf("parse response for %s: %w", symbol, err)
+	}
+
+	// L'API StockTwits retourne un status dans le body même en HTTP 200
+	if data.Response.Status != 0 && data.Response.Status != 200 {
+		return nil, fmt.Errorf("StockTwits API error %d for %s: %s", data.Response.Status, symbol, data.Response.Error)
+	}
+
+	var articles []scraper.Article
+	for _, msg := range data.Messages {
+		if msg.Body == "" {
+			continue
+		}
+		sentiment := ""
+		if msg.Entities.Sentiment != nil {
+			sentiment = " [" + msg.Entities.Sentiment.Basic + "]"
+		}
+		title := fmt.Sprintf("$%s — @%s%s", symbol, msg.User.Username, sentiment)
+		publishedAt, _ := time.Parse(time.RFC3339, msg.CreatedAt)
+		msgURL := fmt.Sprintf("https://stocktwits.com/%s/message/%d", msg.User.Username, msg.ID)
+
+		articles = append(articles, scraper.Article{
+			Title:       title,
+			Content:     msg.Body,
+			URL:         msgURL,
+			PublishedAt: &publishedAt,
+			Symbols:     []string{symbol},
+		})
+	}
+	fmt.Printf("stocktwits %s: %d messages fetched\n", symbol, len(articles))
+	return articles, nil
+}