blomios/Tradarr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add frontend + backend + database to retrieve and compute news from Yahoo

Browse Source
This commit is contained in:
Blomios
2026-04-18 23:53:57 +02:00
parent f9b6d35c49
commit 93668273ff
84 changed files with 15431 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
backend/internal/ai/anthropic.go Normal file
View File

@ -0,0 +1,79 @@
package ai
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
)
type anthropicProvider struct {
apiKey string
model string
client *http.Client
}
func newAnthropic(apiKey, model string) *anthropicProvider {
if model == "" {
model = "claude-sonnet-4-6"
}
return &anthropicProvider{
apiKey: apiKey,
model: model,
client: &http.Client{},
}
}
func (p *anthropicProvider) Name() string { return "anthropic" }
func (p *anthropicProvider) Summarize(ctx context.Context, prompt string) (string, error) {
body := map[string]interface{}{
"model": p.model,
"max_tokens": 4096,
"messages": []map[string]string{
{"role": "user", "content": prompt},
},
}
b, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.anthropic.com/v1/messages", bytes.NewReader(b))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("x-api-key", p.apiKey)
req.Header.Set("anthropic-version", "2023-06-01")
resp, err := p.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("anthropic API error %d: %s", resp.StatusCode, raw)
}
var result struct {
Content []struct {
Text string `json:"text"`
} `json:"content"`
}
if err := json.Unmarshal(raw, &result); err != nil {
return "", err
}
if len(result.Content) == 0 {
return "", nil
}
return result.Content[0].Text, nil
}
func (p *anthropicProvider) ListModels(_ context.Context) ([]string, error) {
return []string{
"claude-opus-4-7",
"claude-sonnet-4-6",
"claude-haiku-4-5-20251001",
}, nil
}

84
backend/internal/ai/gemini.go Normal file
View File

@ -0,0 +1,84 @@
package ai
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
)
type geminiProvider struct {
apiKey string
model string
client *http.Client
}
func newGemini(apiKey, model string) *geminiProvider {
if model == "" {
model = "gemini-2.0-flash"
}
return &geminiProvider{
apiKey: apiKey,
model: model,
client: &http.Client{},
}
}
func (p *geminiProvider) Name() string { return "gemini" }
func (p *geminiProvider) Summarize(ctx context.Context, prompt string) (string, error) {
url := fmt.Sprintf(
"https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent?key=%s",
p.model, p.apiKey,
)
body := map[string]interface{}{
"contents": []map[string]interface{}{
{"parts": []map[string]string{{"text": prompt}}},
},
}
b, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(b))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
resp, err := p.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("gemini API error %d: %s", resp.StatusCode, raw)
}
var result struct {
Candidates []struct {
Content struct {
Parts []struct {
Text string `json:"text"`
} `json:"parts"`
} `json:"content"`
} `json:"candidates"`
}
if err := json.Unmarshal(raw, &result); err != nil {
return "", err
}
if len(result.Candidates) == 0 || len(result.Candidates[0].Content.Parts) == 0 {
return "", nil
}
return result.Candidates[0].Content.Parts[0].Text, nil
}
func (p *geminiProvider) ListModels(_ context.Context) ([]string, error) {
return []string{
"gemini-2.0-flash",
"gemini-2.0-flash-lite",
"gemini-1.5-pro",
"gemini-1.5-flash",
}, nil
}

95
backend/internal/ai/ollama.go Normal file
View File

@ -0,0 +1,95 @@
package ai
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
)
type ollamaProvider struct {
endpoint string
model string
client *http.Client
}
func newOllama(endpoint, model string) *ollamaProvider {
if endpoint == "" {
endpoint = "http://ollama:11434"
}
if model == "" {
model = "llama3"
}
return &ollamaProvider{
endpoint: endpoint,
model: model,
client: &http.Client{},
}
}
func (p *ollamaProvider) Name() string { return "ollama" }
func (p *ollamaProvider) Summarize(ctx context.Context, prompt string) (string, error) {
body := map[string]interface{}{
"model": p.model,
"prompt": prompt,
"stream": false,
"options": map[string]interface{}{
"num_ctx": 32768,
},
}
b, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.endpoint+"/api/generate", bytes.NewReader(b))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/json")
resp, err := p.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("ollama API error %d: %s", resp.StatusCode, raw)
}
var result struct {
Response string `json:"response"`
}
if err := json.Unmarshal(raw, &result); err != nil {
return "", err
}
return result.Response, nil
}
func (p *ollamaProvider) ListModels(ctx context.Context) ([]string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.endpoint+"/api/tags", nil)
if err != nil {
return nil, err
}
resp, err := p.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(resp.Body)
var result struct {
Models []struct {
Name string `json:"name"`
} `json:"models"`
}
if err := json.Unmarshal(raw, &result); err != nil {
return nil, err
}
var models []string
for _, m := range result.Models {
models = append(models, m.Name)
}
return models, nil
}

52
backend/internal/ai/openai.go Normal file
View File

@ -0,0 +1,52 @@
package ai
import (
"context"
openai "github.com/sashabaranov/go-openai"
)
type openAIProvider struct {
client *openai.Client
model string
}
func newOpenAI(apiKey, model string) *openAIProvider {
if model == "" {
model = openai.GPT4oMini
}
return &openAIProvider{
client: openai.NewClient(apiKey),
model: model,
}
}
func (p *openAIProvider) Name() string { return "openai" }
func (p *openAIProvider) Summarize(ctx context.Context, prompt string) (string, error) {
resp, err := p.client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
Model: p.model,
Messages: []openai.ChatCompletionMessage{
{Role: openai.ChatMessageRoleUser, Content: prompt},
},
})
if err != nil {
return "", err
}
if len(resp.Choices) == 0 {
return "", nil
}
return resp.Choices[0].Message.Content, nil
}
func (p *openAIProvider) ListModels(ctx context.Context) ([]string, error) {
resp, err := p.client.ListModels(ctx)
if err != nil {
return nil, err
}
var models []string
for _, m := range resp.Models {
models = append(models, m.ID)
}
return models, nil
}

160
backend/internal/ai/pipeline.go Normal file
View File

@ -0,0 +1,160 @@
package ai
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/tradarr/backend/internal/crypto"
"github.com/tradarr/backend/internal/models"
)
const DefaultSystemPrompt = `Tu es un assistant spécialisé en trading financier. Analyse l'ensemble des actualités suivantes, toutes sources confondues, et crée un résumé global structuré en français, orienté trading.
Structure ton résumé ainsi :
1. **Vue macro** : tendances globales du marché (économie, géopolitique, secteurs)
2. **Actifs surveillés** : pour chaque actif de la watchlist mentionné dans les news :
- Sentiment (haussier/baissier/neutre)
- Faits clés et catalyseurs
- Risques et opportunités
3. **Autres mouvements notables** : actifs hors watchlist à surveiller
4. **Synthèse** : points d'attention prioritaires pour la journée`
type Pipeline struct {
repo *models.Repository
enc *crypto.Encryptor
}
func NewPipeline(repo *models.Repository, enc *crypto.Encryptor) *Pipeline {
return &Pipeline{repo: repo, enc: enc}
}
// BuildProvider instancie un provider à partir de ses paramètres
func (p *Pipeline) BuildProvider(name, apiKey, endpoint string) (Provider, error) {
provider, err := p.repo.GetActiveAIProvider()
if err != nil {
return nil, err
}
model := ""
if provider != nil {
model = provider.Model
}
return NewProvider(name, apiKey, model, endpoint)
}
// GenerateForUser génère un résumé personnalisé pour un utilisateur
func (p *Pipeline) GenerateForUser(ctx context.Context, userID string) (*models.Summary, error) {
// Récupérer le provider actif
providerCfg, err := p.repo.GetActiveAIProvider()
if err != nil {
return nil, fmt.Errorf("get active provider: %w", err)
}
if providerCfg == nil {
return nil, fmt.Errorf("no active AI provider configured")
}
apiKey := ""
if providerCfg.APIKeyEncrypted != "" {
apiKey, err = p.enc.Decrypt(providerCfg.APIKeyEncrypted)
if err != nil {
return nil, fmt.Errorf("decrypt API key: %w", err)
}
}
provider, err := NewProvider(providerCfg.Name, apiKey, providerCfg.Model, providerCfg.Endpoint)
if err != nil {
return nil, fmt.Errorf("build provider: %w", err)
}
// Récupérer la watchlist de l'utilisateur (pour le contexte IA uniquement)
assets, err := p.repo.GetUserAssets(userID)
if err != nil {
return nil, fmt.Errorf("get user assets: %w", err)
}
symbols := make([]string, len(assets))
for i, a := range assets {
symbols[i] = a.Symbol
}
// Récupérer TOUS les articles récents, toutes sources confondues
hoursStr, _ := p.repo.GetSetting("articles_lookback_hours")
hours, _ := strconv.Atoi(hoursStr)
if hours == 0 {
hours = 24
}
articles, err := p.repo.GetRecentArticles(hours)
if err != nil {
return nil, fmt.Errorf("get articles: %w", err)
}
if len(articles) == 0 {
return nil, fmt.Errorf("no recent articles found")
}
maxStr, _ := p.repo.GetSetting("summary_max_articles")
maxArticles, _ := strconv.Atoi(maxStr)
if maxArticles == 0 {
maxArticles = 50
}
if len(articles) > maxArticles {
articles = articles[:maxArticles]
}
systemPrompt, _ := p.repo.GetSetting("ai_system_prompt")
if systemPrompt == "" {
systemPrompt = DefaultSystemPrompt
}
prompt := buildPrompt(systemPrompt, symbols, articles)
summary, err := provider.Summarize(ctx, prompt)
if err != nil {
return nil, fmt.Errorf("AI summarize: %w", err)
}
return p.repo.CreateSummary(userID, summary, &providerCfg.ID)
}
// GenerateForAll génère les résumés pour tous les utilisateurs ayant une watchlist
func (p *Pipeline) GenerateForAll(ctx context.Context) error {
users, err := p.repo.ListUsers()
if err != nil {
return err
}
for _, user := range users {
if _, err := p.GenerateForUser(ctx, user.ID); err != nil {
fmt.Printf("summary for user %s: %v\n", user.Email, err)
}
}
return nil
}
func buildPrompt(systemPrompt string, symbols []string, articles []models.Article) string {
var sb strings.Builder
sb.WriteString(systemPrompt)
sb.WriteString("\n\n")
if len(symbols) > 0 {
sb.WriteString("Le trader surveille particulièrement ces actifs (sois attentif à toute mention) : ")
sb.WriteString(strings.Join(symbols, ", "))
sb.WriteString(".\n\n")
}
sb.WriteString(fmt.Sprintf("Date d'analyse : %s\n\n", time.Now().Format("02/01/2006 15:04")))
sb.WriteString("## Actualités\n\n")
for i, a := range articles {
sb.WriteString(fmt.Sprintf("### [%d] %s\n", i+1, a.Title))
sb.WriteString(fmt.Sprintf("Source : %s\n", a.SourceName))
if a.PublishedAt.Valid {
sb.WriteString(fmt.Sprintf("Date : %s\n", a.PublishedAt.Time.Format("02/01/2006 15:04")))
}
content := a.Content
if len(content) > 1000 {
content = content[:1000] + "..."
}
sb.WriteString(content)
sb.WriteString("\n\n")
}
return sb.String()
}

27
backend/internal/ai/provider.go Normal file
View File

@ -0,0 +1,27 @@
package ai
import (
"context"
"fmt"
)
type Provider interface {
Name() string
Summarize(ctx context.Context, prompt string) (string, error)
ListModels(ctx context.Context) ([]string, error)
}
func NewProvider(name, apiKey, model, endpoint string) (Provider, error) {
switch name {
case "openai":
return newOpenAI(apiKey, model), nil
case "anthropic":
return newAnthropic(apiKey, model), nil
case "gemini":
return newGemini(apiKey, model), nil
case "ollama":
return newOllama(endpoint, model), nil
default:
return nil, fmt.Errorf("unknown provider: %s", name)
}
}

349
backend/internal/api/handlers/admin.go Normal file
View File

@ -0,0 +1,349 @@
package handlers
import (
"fmt"
"net/http"
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/ai"
"github.com/tradarr/backend/internal/httputil"
"github.com/tradarr/backend/internal/models"
)
// ── Credentials ────────────────────────────────────────────────────────────
type credentialsRequest struct {
SourceID string `json:"source_id" binding:"required"`
Username string `json:"username" binding:"required"`
Password string `json:"password"`
}
func (h *Handler) GetCredentials(c *gin.Context) {
sources, err := h.repo.ListSources()
if err != nil {
httputil.InternalError(c, err)
return
}
type credResponse struct {
SourceID string `json:"source_id"`
SourceName string `json:"source_name"`
Username string `json:"username"`
HasPassword bool `json:"has_password"`
}
var result []credResponse
for _, src := range sources {
if src.Type != "bloomberg" {
continue
}
cred, _ := h.repo.GetCredentials(src.ID)
r := credResponse{SourceID: src.ID, SourceName: src.Name}
if cred != nil {
r.Username = cred.Username
r.HasPassword = cred.PasswordEncrypted != ""
}
result = append(result, r)
}
httputil.OK(c, result)
}
func (h *Handler) UpsertCredentials(c *gin.Context) {
var req credentialsRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
encPwd := ""
if req.Password != "" {
var err error
encPwd, err = h.enc.Encrypt(req.Password)
if err != nil {
httputil.InternalError(c, err)
return
}
}
if err := h.repo.UpsertCredentials(req.SourceID, req.Username, encPwd); err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, gin.H{"ok": true})
}
// ── AI Providers ───────────────────────────────────────────────────────────
type aiProviderRequest struct {
Name string `json:"name" binding:"required"`
APIKey string `json:"api_key"`
Model string `json:"model"`
Endpoint string `json:"endpoint"`
}
func (h *Handler) ListAIProviders(c *gin.Context) {
providers, err := h.repo.ListAIProviders()
if err != nil {
httputil.InternalError(c, err)
return
}
// Ne pas exposer les clés chiffrées — juste indiquer si elle existe
type safeProvider struct {
ID string `json:"id"`
Name string `json:"name"`
Model string `json:"model"`
Endpoint string `json:"endpoint"`
IsActive bool `json:"is_active"`
HasKey bool `json:"has_key"`
}
var result []safeProvider
for _, p := range providers {
result = append(result, safeProvider{
ID: p.ID,
Name: p.Name,
Model: p.Model,
Endpoint: p.Endpoint,
IsActive: p.IsActive,
HasKey: p.APIKeyEncrypted != "",
})
}
httputil.OK(c, result)
}
func (h *Handler) CreateAIProvider(c *gin.Context) {
var req aiProviderRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
encKey := ""
if req.APIKey != "" {
var err error
encKey, err = h.enc.Encrypt(req.APIKey)
if err != nil {
httputil.InternalError(c, err)
return
}
}
p, err := h.repo.CreateAIProvider(req.Name, encKey, req.Model, req.Endpoint)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.Created(c, p)
}
func (h *Handler) UpdateAIProvider(c *gin.Context) {
id := c.Param("id")
var req aiProviderRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
existing, err := h.repo.GetAIProviderByID(id)
if err != nil || existing == nil {
httputil.NotFound(c)
return
}
encKey := existing.APIKeyEncrypted
if req.APIKey != "" {
encKey, err = h.enc.Encrypt(req.APIKey)
if err != nil {
httputil.InternalError(c, err)
return
}
}
if err := h.repo.UpdateAIProvider(id, encKey, req.Model, req.Endpoint); err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, gin.H{"ok": true})
}
func (h *Handler) SetActiveAIProvider(c *gin.Context) {
id := c.Param("id")
if err := h.repo.SetActiveAIProvider(id); err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, gin.H{"ok": true})
}
func (h *Handler) DeleteAIProvider(c *gin.Context) {
if err := h.repo.DeleteAIProvider(c.Param("id")); err != nil {
httputil.InternalError(c, err)
return
}
httputil.NoContent(c)
}
func (h *Handler) ListAIModels(c *gin.Context) {
id := c.Param("id")
p, err := h.repo.GetAIProviderByID(id)
if err != nil || p == nil {
httputil.NotFound(c)
return
}
apiKey := ""
if p.APIKeyEncrypted != "" {
apiKey, err = h.enc.Decrypt(p.APIKeyEncrypted)
if err != nil {
httputil.InternalError(c, err)
return
}
}
provider, err := h.pipeline.BuildProvider(p.Name, apiKey, p.Endpoint)
if err != nil {
httputil.InternalError(c, err)
return
}
models, err := provider.ListModels(c.Request.Context())
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, models)
}
// ── Sources ────────────────────────────────────────────────────────────────
func (h *Handler) ListSources(c *gin.Context) {
sources, err := h.repo.ListSources()
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, sources)
}
type updateSourceRequest struct {
Enabled bool `json:"enabled"`
}
func (h *Handler) UpdateSource(c *gin.Context) {
var req updateSourceRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
if err := h.repo.UpdateSource(c.Param("id"), req.Enabled); err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, gin.H{"ok": true})
}
// ── Scrape Jobs ────────────────────────────────────────────────────────────
func (h *Handler) ListScrapeJobs(c *gin.Context) {
jobs, err := h.repo.ListScrapeJobs(100)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, jobs)
}
func (h *Handler) TriggerScrapeJob(c *gin.Context) {
type triggerRequest struct {
SourceID string `json:"source_id" binding:"required"`
}
var req triggerRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
go func() {
if err := h.registry.Run(req.SourceID); err != nil {
fmt.Printf("scrape job error: %v\n", err)
}
}()
c.JSON(http.StatusAccepted, gin.H{"ok": true, "message": "job started"})
}
// ── Settings ───────────────────────────────────────────────────────────────
func (h *Handler) ListSettings(c *gin.Context) {
settings, err := h.repo.ListSettings()
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, settings)
}
type updateSettingsRequest struct {
Settings []models.Setting `json:"settings"`
}
func (h *Handler) UpdateSettings(c *gin.Context) {
var req updateSettingsRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
for _, s := range req.Settings {
if err := h.repo.SetSetting(s.Key, s.Value); err != nil {
httputil.InternalError(c, err)
return
}
}
httputil.OK(c, gin.H{"ok": true})
}
func (h *Handler) GetDefaultSystemPrompt(c *gin.Context) {
httputil.OK(c, gin.H{"prompt": ai.DefaultSystemPrompt})
}
// ── Admin Users ────────────────────────────────────────────────────────────
func (h *Handler) ListUsers(c *gin.Context) {
users, err := h.repo.ListUsers()
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, users)
}
type updateUserRequest struct {
Email string `json:"email" binding:"required,email"`
Role string `json:"role" binding:"required"`
}
func (h *Handler) UpdateAdminUser(c *gin.Context) {
id := c.Param("id")
var req updateUserRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
if req.Role != "admin" && req.Role != "user" {
httputil.BadRequest(c, fmt.Errorf("role must be admin or user"))
return
}
user, err := h.repo.UpdateUser(id, req.Email, models.Role(req.Role))
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, user)
}
func (h *Handler) DeleteAdminUser(c *gin.Context) {
id := c.Param("id")
// Empêcher la suppression du dernier admin
user, err := h.repo.GetUserByID(id)
if err != nil || user == nil {
httputil.NotFound(c)
return
}
if user.Role == "admin" {
count, _ := h.repo.CountAdmins()
if count <= 1 {
c.JSON(http.StatusBadRequest, gin.H{"error": "cannot delete the last admin"})
return
}
}
if err := h.repo.DeleteUser(id); err != nil {
httputil.InternalError(c, err)
return
}
httputil.NoContent(c)
}

36
backend/internal/api/handlers/articles.go Normal file
View File

@ -0,0 +1,36 @@
package handlers
import (
"strconv"
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/httputil"
)
func (h *Handler) ListArticles(c *gin.Context) {
symbol := c.Query("symbol")
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
offset, _ := strconv.Atoi(c.DefaultQuery("offset", "0"))
if limit > 100 {
limit = 100
}
articles, err := h.repo.ListArticles(symbol, limit, offset)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, articles)
}
func (h *Handler) GetArticle(c *gin.Context) {
article, err := h.repo.GetArticleByID(c.Param("id"))
if err != nil {
httputil.InternalError(c, err)
return
}
if article == nil {
httputil.NotFound(c)
return
}
httputil.OK(c, article)
}

64
backend/internal/api/handlers/auth.go Normal file
View File

@ -0,0 +1,64 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/auth"
"github.com/tradarr/backend/internal/httputil"
"github.com/tradarr/backend/internal/models"
)
type loginRequest struct {
Email string `json:"email" binding:"required,email"`
Password string `json:"password" binding:"required"`
}
type registerRequest struct {
Email string `json:"email" binding:"required,email"`
Password string `json:"password" binding:"required,min=6"`
}
func (h *Handler) Login(c *gin.Context) {
var req loginRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
user, err := h.repo.GetUserByEmail(req.Email)
if err != nil || user == nil || !auth.CheckPassword(user.PasswordHash, req.Password) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid credentials"})
return
}
token, err := auth.GenerateToken(user.ID, user.Email, string(user.Role), h.cfg.JWTSecret)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, gin.H{"token": token, "user": user})
}
func (h *Handler) Register(c *gin.Context) {
var req registerRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
existing, _ := h.repo.GetUserByEmail(req.Email)
if existing != nil {
c.JSON(http.StatusConflict, gin.H{"error": "email already in use"})
return
}
hash, err := auth.HashPassword(req.Password)
if err != nil {
httputil.InternalError(c, err)
return
}
user, err := h.repo.CreateUser(req.Email, hash, models.RoleUser)
if err != nil {
httputil.InternalError(c, err)
return
}
token, _ := auth.GenerateToken(user.ID, user.Email, string(user.Role), h.cfg.JWTSecret)
httputil.Created(c, gin.H{"token": token, "user": user})
}

33
backend/internal/api/handlers/handler.go Normal file
View File

@ -0,0 +1,33 @@
package handlers
import (
"github.com/tradarr/backend/internal/ai"
"github.com/tradarr/backend/internal/config"
"github.com/tradarr/backend/internal/crypto"
"github.com/tradarr/backend/internal/models"
"github.com/tradarr/backend/internal/scraper"
)
type Handler struct {
repo *models.Repository
cfg *config.Config
enc *crypto.Encryptor
registry *scraper.Registry
pipeline *ai.Pipeline
}
func New(
repo *models.Repository,
cfg *config.Config,
enc *crypto.Encryptor,
registry *scraper.Registry,
pipeline *ai.Pipeline,
) *Handler {
return &Handler{
repo: repo,
cfg: cfg,
enc: enc,
registry: registry,
pipeline: pipeline,
}
}

33
backend/internal/api/handlers/summaries.go Normal file
View File

@ -0,0 +1,33 @@
package handlers
import (
"context"
"strconv"
"time"
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/httputil"
)
func (h *Handler) ListSummaries(c *gin.Context) {
userID := c.GetString("userID")
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "10"))
summaries, err := h.repo.ListSummaries(userID, limit)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, summaries)
}
func (h *Handler) GenerateSummary(c *gin.Context) {
userID := c.GetString("userID")
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour)
defer cancel()
summary, err := h.pipeline.GenerateForUser(ctx, userID)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.Created(c, summary)
}

56
backend/internal/api/handlers/user.go Normal file
View File

@ -0,0 +1,56 @@
package handlers
import (
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/httputil"
)
func (h *Handler) GetMe(c *gin.Context) {
userID := c.GetString("userID")
user, err := h.repo.GetUserByID(userID)
if err != nil || user == nil {
httputil.NotFound(c)
return
}
httputil.OK(c, user)
}
func (h *Handler) GetMyAssets(c *gin.Context) {
userID := c.GetString("userID")
assets, err := h.repo.GetUserAssets(userID)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.OK(c, assets)
}
type addAssetRequest struct {
Symbol string `json:"symbol" binding:"required"`
Name string `json:"name"`
}
func (h *Handler) AddMyAsset(c *gin.Context) {
userID := c.GetString("userID")
var req addAssetRequest
if err := c.ShouldBindJSON(&req); err != nil {
httputil.BadRequest(c, err)
return
}
asset, err := h.repo.AddUserAsset(userID, req.Symbol, req.Name)
if err != nil {
httputil.InternalError(c, err)
return
}
httputil.Created(c, asset)
}
func (h *Handler) RemoveMyAsset(c *gin.Context) {
userID := c.GetString("userID")
symbol := c.Param("symbol")
if err := h.repo.RemoveUserAsset(userID, symbol); err != nil {
httputil.InternalError(c, err)
return
}
httputil.NoContent(c)
}

73
backend/internal/api/router.go Normal file
View File

@ -0,0 +1,73 @@
package api
import (
"github.com/gin-gonic/gin"
"github.com/tradarr/backend/internal/api/handlers"
"github.com/tradarr/backend/internal/auth"
)
func SetupRouter(h *handlers.Handler, jwtSecret string) *gin.Engine {
r := gin.Default()
r.Use(func(c *gin.Context) {
c.Header("Access-Control-Allow-Origin", "*")
c.Header("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS")
c.Header("Access-Control-Allow-Headers", "Authorization,Content-Type")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)
return
}
c.Next()
})
api := r.Group("/api")
// Auth public
api.POST("/auth/login", h.Login)
api.POST("/auth/register", h.Register)
// Routes authentifiées
authed := api.Group("/")
authed.Use(auth.Middleware(jwtSecret))
authed.GET("/me", h.GetMe)
authed.GET("/me/assets", h.GetMyAssets)
authed.POST("/me/assets", h.AddMyAsset)
authed.DELETE("/me/assets/:symbol", h.RemoveMyAsset)
authed.GET("/articles", h.ListArticles)
authed.GET("/articles/:id", h.GetArticle)
authed.GET("/summaries", h.ListSummaries)
authed.POST("/summaries/generate", h.GenerateSummary)
// Admin
admin := authed.Group("/admin")
admin.Use(auth.AdminOnly())
admin.GET("/credentials", h.GetCredentials)
admin.PUT("/credentials", h.UpsertCredentials)
admin.GET("/ai-providers", h.ListAIProviders)
admin.POST("/ai-providers", h.CreateAIProvider)
admin.PUT("/ai-providers/:id", h.UpdateAIProvider)
admin.POST("/ai-providers/:id/activate", h.SetActiveAIProvider)
admin.DELETE("/ai-providers/:id", h.DeleteAIProvider)
admin.GET("/ai-providers/:id/models", h.ListAIModels)
admin.GET("/sources", h.ListSources)
admin.PUT("/sources/:id", h.UpdateSource)
admin.GET("/scrape-jobs", h.ListScrapeJobs)
admin.POST("/scrape-jobs/trigger", h.TriggerScrapeJob)
admin.GET("/settings", h.ListSettings)
admin.PUT("/settings", h.UpdateSettings)
admin.GET("/settings/default-prompt", h.GetDefaultSystemPrompt)
admin.GET("/users", h.ListUsers)
admin.PUT("/users/:id", h.UpdateAdminUser)
admin.DELETE("/users/:id", h.DeleteAdminUser)
return r
}

46
backend/internal/auth/jwt.go Normal file
View File

@ -0,0 +1,46 @@
package auth
import (
"fmt"
"time"
"github.com/golang-jwt/jwt/v5"
)
type Claims struct {
UserID string `json:"user_id"`
Email string `json:"email"`
Role string `json:"role"`
jwt.RegisteredClaims
}
func GenerateToken(userID, email, role, secret string) (string, error) {
claims := Claims{
UserID: userID,
Email: email,
Role: role,
RegisteredClaims: jwt.RegisteredClaims{
ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)),
IssuedAt: jwt.NewNumericDate(time.Now()),
},
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
return token.SignedString([]byte(secret))
}
func ValidateToken(tokenStr, secret string) (*Claims, error) {
token, err := jwt.ParseWithClaims(tokenStr, &Claims{}, func(t *jwt.Token) (interface{}, error) {
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
return nil, fmt.Errorf("unexpected signing method")
}
return []byte(secret), nil
})
if err != nil {
return nil, err
}
claims, ok := token.Claims.(*Claims)
if !ok || !token.Valid {
return nil, fmt.Errorf("invalid token")
}
return claims, nil
}

37
backend/internal/auth/middleware.go Normal file
View File

@ -0,0 +1,37 @@
package auth
import (
"net/http"
"strings"
"github.com/gin-gonic/gin"
)
func Middleware(secret string) gin.HandlerFunc {
return func(c *gin.Context) {
header := c.GetHeader("Authorization")
if !strings.HasPrefix(header, "Bearer ") {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing token"})
return
}
claims, err := ValidateToken(strings.TrimPrefix(header, "Bearer "), secret)
if err != nil {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid token"})
return
}
c.Set("userID", claims.UserID)
c.Set("email", claims.Email)
c.Set("role", claims.Role)
c.Next()
}
}
func AdminOnly() gin.HandlerFunc {
return func(c *gin.Context) {
if c.GetString("role") != "admin" {
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "admin only"})
return
}
c.Next()
}
}

12
backend/internal/auth/passwords.go Normal file
View File

@ -0,0 +1,12 @@
package auth
import "golang.org/x/crypto/bcrypt"
func HashPassword(password string) (string, error) {
b, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
return string(b), err
}
func CheckPassword(hash, password string) bool {
return bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) == nil
}

53
backend/internal/config/config.go Normal file
View File

@ -0,0 +1,53 @@
package config
import (
"encoding/hex"
"fmt"
"os"
)
type Config struct {
DatabaseURL string
JWTSecret string
EncryptionKey []byte
Port string
ChromePath string
AdminEmail string
AdminPassword string
}
func Load() (*Config, error) {
dbURL := os.Getenv("DATABASE_URL")
if dbURL == "" {
return nil, fmt.Errorf("DATABASE_URL is required")
}
jwtSecret := os.Getenv("JWT_SECRET")
if jwtSecret == "" {
return nil, fmt.Errorf("JWT_SECRET is required")
}
encHex := os.Getenv("ENCRYPTION_KEY")
if encHex == "" {
return nil, fmt.Errorf("ENCRYPTION_KEY is required")
}
encKey, err := hex.DecodeString(encHex)
if err != nil || len(encKey) != 32 {
return nil, fmt.Errorf("ENCRYPTION_KEY must be a valid 32-byte hex string")
}
port := os.Getenv("PORT")
if port == "" {
port = "8080"
}
return &Config{
DatabaseURL: dbURL,
JWTSecret: jwtSecret,
EncryptionKey: encKey,
Port: port,
ChromePath: os.Getenv("CHROME_PATH"),
AdminEmail: os.Getenv("ADMIN_EMAIL"),
AdminPassword: os.Getenv("ADMIN_PASSWORD"),
}, nil
}

59
backend/internal/crypto/aes.go Normal file
View File

@ -0,0 +1,59 @@
package crypto
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"fmt"
"io"
)
type Encryptor struct {
key []byte
}
func New(key []byte) *Encryptor {
return &Encryptor{key: key}
}
func (e *Encryptor) Encrypt(plaintext string) (string, error) {
block, err := aes.NewCipher(e.key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
ciphertext := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return base64.StdEncoding.EncodeToString(ciphertext), nil
}
func (e *Encryptor) Decrypt(encoded string) (string, error) {
data, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return "", err
}
block, err := aes.NewCipher(e.key)
if err != nil {
return "", err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
if len(data) < gcm.NonceSize() {
return "", fmt.Errorf("ciphertext too short")
}
nonce, ciphertext := data[:gcm.NonceSize()], data[gcm.NonceSize():]
plain, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", err
}
return string(plain), nil
}

43
backend/internal/database/db.go Normal file
View File

@ -0,0 +1,43 @@
package database
import (
"database/sql"
"fmt"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
_ "github.com/lib/pq"
)
func Connect(databaseURL string) (*sql.DB, error) {
db, err := sql.Open("postgres", databaseURL)
if err != nil {
return nil, fmt.Errorf("open db: %w", err)
}
if err := db.Ping(); err != nil {
return nil, fmt.Errorf("ping db: %w", err)
}
db.SetMaxOpenConns(25)
db.SetMaxIdleConns(5)
return db, nil
}
func RunMigrations(db *sql.DB) error {
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return fmt.Errorf("migration driver: %w", err)
}
m, err := migrate.NewWithDatabaseInstance(
"file://internal/database/migrations",
"postgres",
driver,
)
if err != nil {
return fmt.Errorf("migrate init: %w", err)
}
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
return fmt.Errorf("migrate up: %w", err)
}
return nil
}

10
backend/internal/database/migrations/000001_init.down.sql Normal file
View File

@ -0,0 +1,10 @@
DROP TABLE IF EXISTS settings CASCADE;
DROP TABLE IF EXISTS summaries CASCADE;
DROP TABLE IF EXISTS ai_providers CASCADE;
DROP TABLE IF EXISTS scrape_jobs CASCADE;
DROP TABLE IF EXISTS scrape_credentials CASCADE;
DROP TABLE IF EXISTS article_symbols CASCADE;
DROP TABLE IF EXISTS articles CASCADE;
DROP TABLE IF EXISTS sources CASCADE;
DROP TABLE IF EXISTS user_assets CASCADE;
DROP TABLE IF EXISTS users CASCADE;

106
backend/internal/database/migrations/000001_init.up.sql Normal file
View File

@ -0,0 +1,106 @@
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
email TEXT NOT NULL UNIQUE,
password_hash TEXT NOT NULL,
role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('admin', 'user')),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE user_assets (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
symbol VARCHAR(20) NOT NULL,
name TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE (user_id, symbol)
);
CREATE TABLE sources (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
type TEXT NOT NULL CHECK (type IN ('bloomberg', 'stocktwits')),
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE articles (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
title TEXT NOT NULL,
content TEXT NOT NULL DEFAULT '',
url TEXT NOT NULL UNIQUE,
published_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE article_symbols (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
article_id UUID NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
symbol VARCHAR(20) NOT NULL,
UNIQUE (article_id, symbol)
);
CREATE TABLE scrape_credentials (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE UNIQUE,
username TEXT NOT NULL DEFAULT '',
password_encrypted TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE scrape_jobs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'done', 'error')),
started_at TIMESTAMPTZ,
finished_at TIMESTAMPTZ,
articles_found INT NOT NULL DEFAULT 0,
error_msg TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE ai_providers (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL CHECK (name IN ('openai', 'anthropic', 'gemini', 'ollama')),
api_key_encrypted TEXT NOT NULL DEFAULT '',
model TEXT NOT NULL DEFAULT '',
endpoint TEXT NOT NULL DEFAULT '',
is_active BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE summaries (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
content TEXT NOT NULL,
ai_provider_id UUID REFERENCES ai_providers(id) ON DELETE SET NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT ''
);
-- Index pour les performances
CREATE INDEX idx_articles_source_id ON articles(source_id);
CREATE INDEX idx_articles_published_at ON articles(published_at DESC);
CREATE INDEX idx_article_symbols_symbol ON article_symbols(symbol);
CREATE INDEX idx_summaries_user_id ON summaries(user_id);
CREATE INDEX idx_summaries_generated_at ON summaries(generated_at DESC);
CREATE INDEX idx_scrape_jobs_status ON scrape_jobs(status);
CREATE INDEX idx_user_assets_user_id ON user_assets(user_id);
-- Sources initiales
INSERT INTO sources (name, type, enabled) VALUES
('Bloomberg', 'bloomberg', TRUE),
('StockTwits', 'stocktwits', TRUE);
-- Paramètres par défaut
INSERT INTO settings (key, value) VALUES
('scrape_interval_minutes', '60'),
('articles_lookback_hours', '24'),
('summary_max_articles', '50');

1
backend/internal/database/migrations/000002_ai_prompt.down.sql Normal file
View File

@ -0,0 +1 @@
DELETE FROM settings WHERE key = 'ai_system_prompt';

13
backend/internal/database/migrations/000002_ai_prompt.up.sql Normal file
View File

@ -0,0 +1,13 @@
INSERT INTO settings (key, value) VALUES (
'ai_system_prompt',
'Tu es un assistant spécialisé en trading financier. Analyse l''ensemble des actualités suivantes, toutes sources confondues, et crée un résumé global structuré en français, orienté trading.
Structure ton résumé ainsi :
1. **Vue macro** : tendances globales du marché (économie, géopolitique, secteurs)
2. **Actifs surveillés** : pour chaque actif de la watchlist mentionné dans les news :
- Sentiment (haussier/baissier/neutre)
- Faits clés et catalyseurs
- Risques et opportunités
3. **Autres mouvements notables** : actifs hors watchlist à surveiller
4. **Synthèse** : points d''attention prioritaires pour la journée'
) ON CONFLICT (key) DO NOTHING;

48
backend/internal/httputil/response.go Normal file
View File

@ -0,0 +1,48 @@
package httputil
import (
"net/http"
"reflect"
"github.com/gin-gonic/gin"
)
// nilToEmpty converts nil slices to empty slices so JSON serializes as [] not null
func nilToEmpty(data interface{}) interface{} {
if data == nil {
return data
}
v := reflect.ValueOf(data)
if v.Kind() == reflect.Slice && v.IsNil() {
return reflect.MakeSlice(v.Type(), 0, 0).Interface()
}
return data
}
func OK(c *gin.Context, data interface{}) {
c.JSON(http.StatusOK, gin.H{"data": nilToEmpty(data)})
}
func Created(c *gin.Context, data interface{}) {
c.JSON(http.StatusCreated, gin.H{"data": nilToEmpty(data)})
}
func NoContent(c *gin.Context) {
c.Status(http.StatusNoContent)
}
func BadRequest(c *gin.Context, err error) {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
}
func Unauthorized(c *gin.Context) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
}
func NotFound(c *gin.Context) {
c.JSON(http.StatusNotFound, gin.H{"error": "not found"})
}
func InternalError(c *gin.Context, err error) {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
}

99
backend/internal/models/models.go Normal file
View File

@ -0,0 +1,99 @@
package models
import (
"database/sql"
"time"
)
type Role string
const (
RoleAdmin Role = "admin"
RoleUser Role = "user"
)
type User struct {
ID string `json:"id"`
Email string `json:"email"`
PasswordHash string `json:"-"`
Role Role `json:"role"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type UserAsset struct {
ID string `json:"id"`
UserID string `json:"user_id"`
Symbol string `json:"symbol"`
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
}
type Source struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
}
type Article struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
SourceName string `json:"source_name,omitempty"`
Title string `json:"title"`
Content string `json:"content"`
URL string `json:"url"`
PublishedAt sql.NullTime `json:"published_at"`
CreatedAt time.Time `json:"created_at"`
Symbols []string `json:"symbols,omitempty"`
}
type ArticleSymbol struct {
ID string `json:"id"`
ArticleID string `json:"article_id"`
Symbol string `json:"symbol"`
}
type ScrapeCredential struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
Username string `json:"username"`
PasswordEncrypted string `json:"-"`
UpdatedAt time.Time `json:"updated_at"`
}
type ScrapeJob struct {
ID string `json:"id"`
SourceID string `json:"source_id"`
SourceName string `json:"source_name,omitempty"`
Status string `json:"status"`
StartedAt sql.NullTime `json:"started_at"`
FinishedAt sql.NullTime `json:"finished_at"`
ArticlesFound int `json:"articles_found"`
ErrorMsg string `json:"error_msg"`
CreatedAt time.Time `json:"created_at"`
}
type AIProvider struct {
ID string `json:"id"`
Name string `json:"name"`
APIKeyEncrypted string `json:"-"`
Model string `json:"model"`
Endpoint string `json:"endpoint"`
IsActive bool `json:"is_active"`
CreatedAt time.Time `json:"created_at"`
}
type Summary struct {
ID string `json:"id"`
UserID string `json:"user_id"`
Content string `json:"content"`
AIProviderID *string `json:"ai_provider_id"`
GeneratedAt time.Time `json:"generated_at"`
}
type Setting struct {
Key string `json:"key"`
Value string `json:"value"`
}

538
backend/internal/models/repository.go Normal file
View File

@ -0,0 +1,538 @@
package models
import (
"database/sql"
"fmt"
"strings"
"time"
)
type Repository struct {
db *sql.DB
}
func NewRepository(db *sql.DB) *Repository {
return &Repository{db: db}
}
// ── Users ──────────────────────────────────────────────────────────────────
func (r *Repository) CreateUser(email, passwordHash string, role Role) (*User, error) {
u := &User{}
err := r.db.QueryRow(`
INSERT INTO users (email, password_hash, role)
VALUES ($1, $2, $3)
RETURNING id, email, password_hash, role, created_at, updated_at`,
email, passwordHash, role,
).Scan(&u.ID, &u.Email, &u.PasswordHash, &u.Role, &u.CreatedAt, &u.UpdatedAt)
return u, err
}
func (r *Repository) GetUserByEmail(email string) (*User, error) {
u := &User{}
err := r.db.QueryRow(`
SELECT id, email, password_hash, role, created_at, updated_at
FROM users WHERE email = $1`, email,
).Scan(&u.ID, &u.Email, &u.PasswordHash, &u.Role, &u.CreatedAt, &u.UpdatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return u, err
}
func (r *Repository) GetUserByID(id string) (*User, error) {
u := &User{}
err := r.db.QueryRow(`
SELECT id, email, password_hash, role, created_at, updated_at
FROM users WHERE id = $1`, id,
).Scan(&u.ID, &u.Email, &u.PasswordHash, &u.Role, &u.CreatedAt, &u.UpdatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return u, err
}
func (r *Repository) ListUsers() ([]User, error) {
rows, err := r.db.Query(`
SELECT id, email, password_hash, role, created_at, updated_at
FROM users ORDER BY created_at DESC`)
if err != nil {
return nil, err
}
defer rows.Close()
var users []User
for rows.Next() {
var u User
if err := rows.Scan(&u.ID, &u.Email, &u.PasswordHash, &u.Role, &u.CreatedAt, &u.UpdatedAt); err != nil {
return nil, err
}
users = append(users, u)
}
return users, nil
}
func (r *Repository) UpdateUser(id, email string, role Role) (*User, error) {
u := &User{}
err := r.db.QueryRow(`
UPDATE users SET email=$1, role=$2, updated_at=NOW()
WHERE id=$3
RETURNING id, email, password_hash, role, created_at, updated_at`,
email, role, id,
).Scan(&u.ID, &u.Email, &u.PasswordHash, &u.Role, &u.CreatedAt, &u.UpdatedAt)
return u, err
}
func (r *Repository) DeleteUser(id string) error {
_, err := r.db.Exec(`DELETE FROM users WHERE id=$1`, id)
return err
}
func (r *Repository) CountAdmins() (int, error) {
var count int
err := r.db.QueryRow(`SELECT COUNT(*) FROM users WHERE role='admin'`).Scan(&count)
return count, err
}
// ── User Assets ────────────────────────────────────────────────────────────
func (r *Repository) GetUserAssets(userID string) ([]UserAsset, error) {
rows, err := r.db.Query(`
SELECT id, user_id, symbol, name, created_at
FROM user_assets WHERE user_id=$1 ORDER BY symbol`, userID)
if err != nil {
return nil, err
}
defer rows.Close()
var assets []UserAsset
for rows.Next() {
var a UserAsset
if err := rows.Scan(&a.ID, &a.UserID, &a.Symbol, &a.Name, &a.CreatedAt); err != nil {
return nil, err
}
assets = append(assets, a)
}
return assets, nil
}
func (r *Repository) AddUserAsset(userID, symbol, name string) (*UserAsset, error) {
a := &UserAsset{}
err := r.db.QueryRow(`
INSERT INTO user_assets (user_id, symbol, name)
VALUES ($1, $2, $3)
ON CONFLICT (user_id, symbol) DO UPDATE SET name=EXCLUDED.name
RETURNING id, user_id, symbol, name, created_at`,
userID, strings.ToUpper(symbol), name,
).Scan(&a.ID, &a.UserID, &a.Symbol, &a.Name, &a.CreatedAt)
return a, err
}
func (r *Repository) RemoveUserAsset(userID, symbol string) error {
_, err := r.db.Exec(`
DELETE FROM user_assets WHERE user_id=$1 AND symbol=$2`,
userID, strings.ToUpper(symbol))
return err
}
func (r *Repository) GetAllWatchedSymbols() ([]string, error) {
rows, err := r.db.Query(`SELECT DISTINCT symbol FROM user_assets ORDER BY symbol`)
if err != nil {
return nil, err
}
defer rows.Close()
var symbols []string
for rows.Next() {
var s string
if err := rows.Scan(&s); err != nil {
return nil, err
}
symbols = append(symbols, s)
}
return symbols, nil
}
// ── Sources ────────────────────────────────────────────────────────────────
func (r *Repository) ListSources() ([]Source, error) {
rows, err := r.db.Query(`SELECT id, name, type, enabled, created_at FROM sources ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
var sources []Source
for rows.Next() {
var s Source
if err := rows.Scan(&s.ID, &s.Name, &s.Type, &s.Enabled, &s.CreatedAt); err != nil {
return nil, err
}
sources = append(sources, s)
}
return sources, nil
}
func (r *Repository) GetSourceByType(sourceType string) (*Source, error) {
s := &Source{}
err := r.db.QueryRow(`
SELECT id, name, type, enabled, created_at FROM sources WHERE type=$1`, sourceType,
).Scan(&s.ID, &s.Name, &s.Type, &s.Enabled, &s.CreatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return s, err
}
func (r *Repository) UpdateSource(id string, enabled bool) error {
_, err := r.db.Exec(`UPDATE sources SET enabled=$1 WHERE id=$2`, enabled, id)
return err
}
// ── Articles ───────────────────────────────────────────────────────────────
func (r *Repository) UpsertArticle(sourceID, title, content, url string, publishedAt *time.Time) (*Article, error) {
a := &Article{}
var pa sql.NullTime
if publishedAt != nil {
pa = sql.NullTime{Time: *publishedAt, Valid: true}
}
err := r.db.QueryRow(`
INSERT INTO articles (source_id, title, content, url, published_at)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (url) DO UPDATE SET title=EXCLUDED.title, content=EXCLUDED.content
RETURNING id, source_id, title, content, url, published_at, created_at`,
sourceID, title, content, url, pa,
).Scan(&a.ID, &a.SourceID, &a.Title, &a.Content, &a.URL, &a.PublishedAt, &a.CreatedAt)
return a, err
}
func (r *Repository) AddArticleSymbol(articleID, symbol string) error {
_, err := r.db.Exec(`
INSERT INTO article_symbols (article_id, symbol)
VALUES ($1, $2) ON CONFLICT DO NOTHING`,
articleID, strings.ToUpper(symbol))
return err
}
func (r *Repository) ListArticles(symbol string, limit, offset int) ([]Article, error) {
query := `
SELECT a.id, a.source_id, s.name, a.title, a.content, a.url, a.published_at, a.created_at
FROM articles a
JOIN sources s ON s.id = a.source_id`
args := []interface{}{}
if symbol != "" {
query += `
JOIN article_symbols asy ON asy.article_id = a.id AND asy.symbol = $1`
args = append(args, strings.ToUpper(symbol))
}
query += ` ORDER BY a.published_at DESC NULLS LAST, a.created_at DESC`
query += fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
args = append(args, limit, offset)
rows, err := r.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var articles []Article
for rows.Next() {
var a Article
if err := rows.Scan(&a.ID, &a.SourceID, &a.SourceName, &a.Title, &a.Content, &a.URL, &a.PublishedAt, &a.CreatedAt); err != nil {
return nil, err
}
articles = append(articles, a)
}
return articles, nil
}
func (r *Repository) GetArticleByID(id string) (*Article, error) {
a := &Article{}
err := r.db.QueryRow(`
SELECT a.id, a.source_id, s.name, a.title, a.content, a.url, a.published_at, a.created_at
FROM articles a JOIN sources s ON s.id=a.source_id
WHERE a.id=$1`, id,
).Scan(&a.ID, &a.SourceID, &a.SourceName, &a.Title, &a.Content, &a.URL, &a.PublishedAt, &a.CreatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return a, err
}
func (r *Repository) GetRecentArticles(hours int) ([]Article, error) {
rows, err := r.db.Query(`
SELECT a.id, a.source_id, s.name, a.title, a.content, a.url, a.published_at, a.created_at
FROM articles a
JOIN sources s ON s.id = a.source_id
WHERE a.created_at > NOW() - ($1 * INTERVAL '1 hour')
ORDER BY a.published_at DESC NULLS LAST, a.created_at DESC`, hours)
if err != nil {
return nil, err
}
defer rows.Close()
var articles []Article
for rows.Next() {
var a Article
if err := rows.Scan(&a.ID, &a.SourceID, &a.SourceName, &a.Title, &a.Content, &a.URL, &a.PublishedAt, &a.CreatedAt); err != nil {
return nil, err
}
articles = append(articles, a)
}
return articles, nil
}
func (r *Repository) GetRecentArticlesForSymbols(symbols []string, hours int) ([]Article, error) {
if len(symbols) == 0 {
return nil, nil
}
placeholders := make([]string, len(symbols))
args := []interface{}{hours}
for i, s := range symbols {
placeholders[i] = fmt.Sprintf("$%d", i+2)
args = append(args, strings.ToUpper(s))
}
query := fmt.Sprintf(`
SELECT DISTINCT a.id, a.source_id, s.name, a.title, a.content, a.url, a.published_at, a.created_at
FROM articles a
JOIN sources s ON s.id = a.source_id
JOIN article_symbols asy ON asy.article_id = a.id
WHERE asy.symbol IN (%s)
AND a.created_at > NOW() - ($1 * INTERVAL '1 hour')
ORDER BY a.published_at DESC NULLS LAST, a.created_at DESC`,
strings.Join(placeholders, ","))
rows, err := r.db.Query(query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var articles []Article
for rows.Next() {
var a Article
if err := rows.Scan(&a.ID, &a.SourceID, &a.SourceName, &a.Title, &a.Content, &a.URL, &a.PublishedAt, &a.CreatedAt); err != nil {
return nil, err
}
articles = append(articles, a)
}
return articles, nil
}
// ── Scrape Credentials ─────────────────────────────────────────────────────
func (r *Repository) GetCredentials(sourceID string) (*ScrapeCredential, error) {
c := &ScrapeCredential{}
err := r.db.QueryRow(`
SELECT id, source_id, username, password_encrypted, updated_at
FROM scrape_credentials WHERE source_id=$1`, sourceID,
).Scan(&c.ID, &c.SourceID, &c.Username, &c.PasswordEncrypted, &c.UpdatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return c, err
}
func (r *Repository) UpsertCredentials(sourceID, username, passwordEncrypted string) error {
_, err := r.db.Exec(`
INSERT INTO scrape_credentials (source_id, username, password_encrypted)
VALUES ($1, $2, $3)
ON CONFLICT (source_id) DO UPDATE
SET username=EXCLUDED.username, password_encrypted=EXCLUDED.password_encrypted, updated_at=NOW()`,
sourceID, username, passwordEncrypted)
return err
}
// ── Scrape Jobs ────────────────────────────────────────────────────────────
func (r *Repository) CreateScrapeJob(sourceID string) (*ScrapeJob, error) {
j := &ScrapeJob{}
err := r.db.QueryRow(`
INSERT INTO scrape_jobs (source_id) VALUES ($1)
RETURNING id, source_id, status, started_at, finished_at, articles_found, error_msg, created_at`,
sourceID,
).Scan(&j.ID, &j.SourceID, &j.Status, &j.StartedAt, &j.FinishedAt, &j.ArticlesFound, &j.ErrorMsg, &j.CreatedAt)
return j, err
}
func (r *Repository) UpdateScrapeJob(id, status string, articlesFound int, errMsg string) error {
var finishedAt *time.Time
if status == "done" || status == "error" {
now := time.Now()
finishedAt = &now
}
_, err := r.db.Exec(`
UPDATE scrape_jobs
SET status=$1, articles_found=$2, error_msg=$3, finished_at=$4,
started_at=CASE WHEN status='pending' THEN NOW() ELSE started_at END
WHERE id=$5`,
status, articlesFound, errMsg, finishedAt, id)
return err
}
func (r *Repository) ListScrapeJobs(limit int) ([]ScrapeJob, error) {
rows, err := r.db.Query(`
SELECT j.id, j.source_id, s.name, j.status, j.started_at, j.finished_at,
j.articles_found, j.error_msg, j.created_at
FROM scrape_jobs j JOIN sources s ON s.id=j.source_id
ORDER BY j.created_at DESC LIMIT $1`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var jobs []ScrapeJob
for rows.Next() {
var j ScrapeJob
if err := rows.Scan(&j.ID, &j.SourceID, &j.SourceName, &j.Status, &j.StartedAt,
&j.FinishedAt, &j.ArticlesFound, &j.ErrorMsg, &j.CreatedAt); err != nil {
return nil, err
}
jobs = append(jobs, j)
}
return jobs, nil
}
// ── AI Providers ───────────────────────────────────────────────────────────
func (r *Repository) ListAIProviders() ([]AIProvider, error) {
rows, err := r.db.Query(`
SELECT id, name, api_key_encrypted, model, endpoint, is_active, created_at
FROM ai_providers ORDER BY created_at`)
if err != nil {
return nil, err
}
defer rows.Close()
var providers []AIProvider
for rows.Next() {
var p AIProvider
if err := rows.Scan(&p.ID, &p.Name, &p.APIKeyEncrypted, &p.Model, &p.Endpoint, &p.IsActive, &p.CreatedAt); err != nil {
return nil, err
}
providers = append(providers, p)
}
return providers, nil
}
func (r *Repository) GetAIProviderByID(id string) (*AIProvider, error) {
p := &AIProvider{}
err := r.db.QueryRow(`
SELECT id, name, api_key_encrypted, model, endpoint, is_active, created_at
FROM ai_providers WHERE id=$1`, id,
).Scan(&p.ID, &p.Name, &p.APIKeyEncrypted, &p.Model, &p.Endpoint, &p.IsActive, &p.CreatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return p, err
}
func (r *Repository) GetActiveAIProvider() (*AIProvider, error) {
p := &AIProvider{}
err := r.db.QueryRow(`
SELECT id, name, api_key_encrypted, model, endpoint, is_active, created_at
FROM ai_providers WHERE is_active=TRUE LIMIT 1`,
).Scan(&p.ID, &p.Name, &p.APIKeyEncrypted, &p.Model, &p.Endpoint, &p.IsActive, &p.CreatedAt)
if err == sql.ErrNoRows {
return nil, nil
}
return p, err
}
func (r *Repository) CreateAIProvider(name, apiKeyEncrypted, model, endpoint string) (*AIProvider, error) {
p := &AIProvider{}
err := r.db.QueryRow(`
INSERT INTO ai_providers (name, api_key_encrypted, model, endpoint)
VALUES ($1, $2, $3, $4)
RETURNING id, name, api_key_encrypted, model, endpoint, is_active, created_at`,
name, apiKeyEncrypted, model, endpoint,
).Scan(&p.ID, &p.Name, &p.APIKeyEncrypted, &p.Model, &p.Endpoint, &p.IsActive, &p.CreatedAt)
return p, err
}
func (r *Repository) UpdateAIProvider(id, apiKeyEncrypted, model, endpoint string) error {
_, err := r.db.Exec(`
UPDATE ai_providers SET api_key_encrypted=$1, model=$2, endpoint=$3 WHERE id=$4`,
apiKeyEncrypted, model, endpoint, id)
return err
}
func (r *Repository) SetActiveAIProvider(id string) error {
tx, err := r.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.Exec(`UPDATE ai_providers SET is_active=FALSE`); err != nil {
return err
}
if _, err := tx.Exec(`UPDATE ai_providers SET is_active=TRUE WHERE id=$1`, id); err != nil {
return err
}
return tx.Commit()
}
func (r *Repository) DeleteAIProvider(id string) error {
_, err := r.db.Exec(`DELETE FROM ai_providers WHERE id=$1`, id)
return err
}
// ── Summaries ──────────────────────────────────────────────────────────────
func (r *Repository) CreateSummary(userID, content string, providerID *string) (*Summary, error) {
s := &Summary{}
err := r.db.QueryRow(`
INSERT INTO summaries (user_id, content, ai_provider_id)
VALUES ($1, $2, $3)
RETURNING id, user_id, content, ai_provider_id, generated_at`,
userID, content, providerID,
).Scan(&s.ID, &s.UserID, &s.Content, &s.AIProviderID, &s.GeneratedAt)
return s, err
}
func (r *Repository) ListSummaries(userID string, limit int) ([]Summary, error) {
rows, err := r.db.Query(`
SELECT id, user_id, content, ai_provider_id, generated_at
FROM summaries WHERE user_id=$1
ORDER BY generated_at DESC LIMIT $2`, userID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var summaries []Summary
for rows.Next() {
var s Summary
if err := rows.Scan(&s.ID, &s.UserID, &s.Content, &s.AIProviderID, &s.GeneratedAt); err != nil {
return nil, err
}
summaries = append(summaries, s)
}
return summaries, nil
}
// ── Settings ───────────────────────────────────────────────────────────────
func (r *Repository) GetSetting(key string) (string, error) {
var value string
err := r.db.QueryRow(`SELECT value FROM settings WHERE key=$1`, key).Scan(&value)
if err == sql.ErrNoRows {
return "", nil
}
return value, err
}
func (r *Repository) SetSetting(key, value string) error {
_, err := r.db.Exec(`
INSERT INTO settings (key, value) VALUES ($1, $2)
ON CONFLICT (key) DO UPDATE SET value=EXCLUDED.value`,
key, value)
return err
}
func (r *Repository) ListSettings() ([]Setting, error) {
rows, err := r.db.Query(`SELECT key, value FROM settings ORDER BY key`)
if err != nil {
return nil, err
}
defer rows.Close()
var settings []Setting
for rows.Next() {
var s Setting
if err := rows.Scan(&s.Key, &s.Value); err != nil {
return nil, err
}
settings = append(settings, s)
}
return settings, nil
}

88
backend/internal/scheduler/scheduler.go Normal file
View File

@ -0,0 +1,88 @@
package scheduler
import (
"context"
"fmt"
"strconv"
"github.com/robfig/cron/v3"
"github.com/tradarr/backend/internal/ai"
"github.com/tradarr/backend/internal/models"
"github.com/tradarr/backend/internal/scraper"
)
type Scheduler struct {
cron *cron.Cron
registry *scraper.Registry
pipeline *ai.Pipeline
repo *models.Repository
entryID cron.EntryID
}
func New(registry *scraper.Registry, pipeline *ai.Pipeline, repo *models.Repository) *Scheduler {
return &Scheduler{
cron: cron.New(),
registry: registry,
pipeline: pipeline,
repo: repo,
}
}
func (s *Scheduler) Start() error {
interval, err := s.getInterval()
if err != nil {
return err
}
spec := fmt.Sprintf("@every %dm", interval)
s.entryID, err = s.cron.AddFunc(spec, s.run)
if err != nil {
return fmt.Errorf("add cron: %w", err)
}
s.cron.Start()
fmt.Printf("scheduler started, running every %d minutes\n", interval)
return nil
}
func (s *Scheduler) Stop() {
s.cron.Stop()
}
func (s *Scheduler) Reload() error {
s.cron.Remove(s.entryID)
interval, err := s.getInterval()
if err != nil {
return err
}
spec := fmt.Sprintf("@every %dm", interval)
s.entryID, err = s.cron.AddFunc(spec, s.run)
return err
}
func (s *Scheduler) run() {
fmt.Println("scheduler: running scraping cycle")
if err := s.registry.RunAll(); err != nil {
fmt.Printf("scheduler scrape error: %v\n", err)
return
}
fmt.Println("scheduler: running AI summaries")
if err := s.pipeline.GenerateForAll(context.Background()); err != nil {
fmt.Printf("scheduler summary error: %v\n", err)
}
}
func (s *Scheduler) getInterval() (int, error) {
v, err := s.repo.GetSetting("scrape_interval_minutes")
if err != nil {
return 60, nil
}
if v == "" {
return 60, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 60, nil
}
return n, nil
}

206
backend/internal/scraper/bloomberg/bloomberg.go Normal file
View File

@ -0,0 +1,206 @@
package bloomberg
import (
"context"
"fmt"
"strings"
"time"
"github.com/chromedp/chromedp"
"github.com/tradarr/backend/internal/scraper"
)
type Bloomberg struct {
username string
password string
chromePath string
}
func New(username, password, chromePath string) *Bloomberg {
return &Bloomberg{username: username, password: password, chromePath: chromePath}
}
func (b *Bloomberg) Name() string { return "bloomberg" }
func (b *Bloomberg) Scrape(ctx context.Context, symbols []string) ([]scraper.Article, error) {
if b.username == "" || b.password == "" {
return nil, fmt.Errorf("bloomberg credentials not configured")
}
opts := []chromedp.ExecAllocatorOption{
chromedp.NoFirstRun,
chromedp.NoDefaultBrowserCheck,
chromedp.Headless,
chromedp.DisableGPU,
chromedp.Flag("no-sandbox", true),
chromedp.Flag("disable-setuid-sandbox", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("disable-blink-features", "AutomationControlled"),
chromedp.Flag("disable-infobars", true),
chromedp.Flag("window-size", "1920,1080"),
chromedp.Flag("ignore-certificate-errors", true),
chromedp.UserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"),
}
if b.chromePath != "" {
opts = append(opts, chromedp.ExecPath(b.chromePath))
}
allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
defer cancelAlloc()
chromeCtx, cancelChrome := chromedp.NewContext(allocCtx)
defer cancelChrome()
timeoutCtx, cancelTimeout := context.WithTimeout(chromeCtx, 5*time.Minute)
defer cancelTimeout()
if err := b.login(timeoutCtx); err != nil {
return nil, fmt.Errorf("bloomberg login: %w", err)
}
var articles []scraper.Article
pages := []string{
"https://www.bloomberg.com/markets",
"https://www.bloomberg.com/technology",
"https://www.bloomberg.com/economics",
}
for _, u := range pages {
pageArticles, err := b.scrapePage(timeoutCtx, u, symbols)
if err != nil {
fmt.Printf("bloomberg scrape %s: %v\n", u, err)
continue
}
articles = append(articles, pageArticles...)
}
fmt.Printf("bloomberg: %d articles fetched total\n", len(articles))
return articles, nil
}
func (b *Bloomberg) login(ctx context.Context) error {
loginCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
// Masquer la détection d'automation via JS
if err := chromedp.Run(loginCtx,
chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.Evaluate(`
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
window.chrome = { runtime: {} };
`, nil).Do(ctx)
}),
); err != nil {
fmt.Printf("bloomberg: could not inject stealth JS: %v\n", err)
}
err := chromedp.Run(loginCtx,
chromedp.Navigate("https://www.bloomberg.com/account/signin"),
chromedp.Sleep(2*time.Second),
// Essayer plusieurs sélecteurs pour l'email
chromedp.ActionFunc(func(ctx context.Context) error {
selectors := []string{
`input[name="email"]`,
`input[type="email"]`,
`input[data-type="email"]`,
`input[placeholder*="email" i]`,
`input[placeholder*="mail" i]`,
}
for _, sel := range selectors {
var count int
if err := chromedp.Evaluate(fmt.Sprintf(`document.querySelectorAll('%s').length`, sel), &count).Do(ctx); err == nil && count > 0 {
fmt.Printf("bloomberg: using email selector: %s\n", sel)
return chromedp.SendKeys(sel, b.username, chromedp.ByQuery).Do(ctx)
}
}
return fmt.Errorf("could not find email input — Bloomberg login page structure may have changed")
}),
chromedp.Sleep(500*time.Millisecond),
// Submit email
chromedp.ActionFunc(func(ctx context.Context) error {
selectors := []string{`button[type="submit"]`, `input[type="submit"]`, `button[data-testid*="submit"]`}
for _, sel := range selectors {
var count int
if err := chromedp.Evaluate(fmt.Sprintf(`document.querySelectorAll('%s').length`, sel), &count).Do(ctx); err == nil && count > 0 {
return chromedp.Click(sel, chromedp.ByQuery).Do(ctx)
}
}
// Fallback: press Enter
return chromedp.KeyEvent("\r").Do(ctx)
}),
chromedp.Sleep(2*time.Second),
// Password
chromedp.ActionFunc(func(ctx context.Context) error {
selectors := []string{`input[type="password"]`, `input[name="password"]`}
for _, sel := range selectors {
var count int
if err := chromedp.Evaluate(fmt.Sprintf(`document.querySelectorAll('%s').length`, sel), &count).Do(ctx); err == nil && count > 0 {
fmt.Printf("bloomberg: using password selector: %s\n", sel)
return chromedp.SendKeys(sel, b.password, chromedp.ByQuery).Do(ctx)
}
}
return fmt.Errorf("could not find password input")
}),
chromedp.Sleep(500*time.Millisecond),
chromedp.ActionFunc(func(ctx context.Context) error {
selectors := []string{`button[type="submit"]`, `input[type="submit"]`}
for _, sel := range selectors {
var count int
if err := chromedp.Evaluate(fmt.Sprintf(`document.querySelectorAll('%s').length`, sel), &count).Do(ctx); err == nil && count > 0 {
return chromedp.Click(sel, chromedp.ByQuery).Do(ctx)
}
}
return chromedp.KeyEvent("\r").Do(ctx)
}),
chromedp.Sleep(3*time.Second),
)
return err
}
func (b *Bloomberg) scrapePage(ctx context.Context, pageURL string, symbols []string) ([]scraper.Article, error) {
pageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
var articleNodes []map[string]string
err := chromedp.Run(pageCtx,
chromedp.Navigate(pageURL),
chromedp.Sleep(3*time.Second),
chromedp.Evaluate(`
(function() {
var items = [];
var seen = new Set();
var links = document.querySelectorAll('a[href*="/news/articles"], a[href*="/opinion/"], a[href*="/markets/"]');
links.forEach(function(a) {
if (seen.has(a.href)) return;
seen.add(a.href);
var title = a.querySelector('h1,h2,h3,h4,[class*="headline"],[class*="title"]');
var text = title ? title.innerText.trim() : a.innerText.trim();
if (text.length > 20 && a.href.includes('bloomberg.com')) {
items.push({title: text, url: a.href});
}
});
return items.slice(0, 25);
})()
`, &articleNodes),
)
if err != nil {
return nil, fmt.Errorf("navigate %s: %w", pageURL, err)
}
var articles []scraper.Article
now := time.Now()
for _, node := range articleNodes {
title := strings.TrimSpace(node["title"])
url := node["url"]
if title == "" || url == "" || !strings.Contains(url, "bloomberg.com") {
continue
}
syms := scraper.DetectSymbols(title, symbols)
articles = append(articles, scraper.Article{
Title: title,
Content: title, // contenu minimal — l'article complet nécessite un accès payant
URL: url,
PublishedAt: &now,
Symbols: syms,
})
}
return articles, nil
}

50
backend/internal/scraper/bloomberg/dynamic.go Normal file
View File

@ -0,0 +1,50 @@
package bloomberg
import (
"context"
"fmt"
"github.com/tradarr/backend/internal/crypto"
"github.com/tradarr/backend/internal/models"
"github.com/tradarr/backend/internal/scraper"
)
// DynamicBloomberg charge les credentials depuis la DB avant chaque scraping
type DynamicBloomberg struct {
repo *models.Repository
enc *crypto.Encryptor
chromePath string
}
func NewDynamic(repo *models.Repository, enc *crypto.Encryptor, chromePath string) *DynamicBloomberg {
return &DynamicBloomberg{repo: repo, enc: enc, chromePath: chromePath}
}
func (d *DynamicBloomberg) Name() string { return "bloomberg" }
func (d *DynamicBloomberg) Scrape(ctx context.Context, symbols []string) ([]scraper.Article, error) {
// Récupérer la source Bloomberg
source, err := d.repo.GetSourceByType("bloomberg")
if err != nil || source == nil {
return nil, fmt.Errorf("bloomberg source not found")
}
cred, err := d.repo.GetCredentials(source.ID)
if err != nil {
return nil, fmt.Errorf("get bloomberg credentials: %w", err)
}
if cred == nil || cred.Username == "" {
return nil, fmt.Errorf("bloomberg credentials not configured — please set them in the admin panel")
}
password := ""
if cred.PasswordEncrypted != "" {
password, err = d.enc.Decrypt(cred.PasswordEncrypted)
if err != nil {
return nil, fmt.Errorf("decrypt bloomberg password: %w", err)
}
}
b := New(cred.Username, password, d.chromePath)
return b.Scrape(ctx, symbols)
}

106
backend/internal/scraper/registry.go Normal file
View File

@ -0,0 +1,106 @@
package scraper
import (
"context"
"fmt"
"time"
"github.com/tradarr/backend/internal/models"
)
type Registry struct {
scrapers map[string]Scraper
repo *models.Repository
}
func NewRegistry(repo *models.Repository) *Registry {
return &Registry{
scrapers: map[string]Scraper{},
repo: repo,
}
}
func (r *Registry) Register(s Scraper) {
r.scrapers[s.Name()] = s
}
// Run exécute le scraper associé à sourceID et persiste les articles
func (r *Registry) Run(sourceID string) error {
sources, err := r.repo.ListSources()
if err != nil {
return err
}
var source *models.Source
for i := range sources {
if sources[i].ID == sourceID {
source = &sources[i]
break
}
}
if source == nil {
return fmt.Errorf("source %s not found", sourceID)
}
scrpr, ok := r.scrapers[source.Type]
if !ok {
return fmt.Errorf("no scraper for type %s", source.Type)
}
// Créer le job
job, err := r.repo.CreateScrapeJob(sourceID)
if err != nil {
return err
}
if err := r.repo.UpdateScrapeJob(job.ID, "running", 0, ""); err != nil {
return err
}
// Récupérer les symboles surveillés
symbols, err := r.repo.GetAllWatchedSymbols()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
articles, scrapeErr := scrpr.Scrape(ctx, symbols)
if scrapeErr != nil {
_ = r.repo.UpdateScrapeJob(job.ID, "error", 0, scrapeErr.Error())
return scrapeErr
}
// Persister les articles
count := 0
for _, a := range articles {
saved, err := r.repo.UpsertArticle(sourceID, a.Title, a.Content, a.URL, a.PublishedAt)
if err != nil {
continue
}
count++
for _, sym := range a.Symbols {
_ = r.repo.AddArticleSymbol(saved.ID, sym)
}
}
return r.repo.UpdateScrapeJob(job.ID, "done", count, "")
}
// RunAll exécute tous les scrapers activés
func (r *Registry) RunAll() error {
sources, err := r.repo.ListSources()
if err != nil {
return err
}
for _, src := range sources {
if !src.Enabled {
continue
}
if err := r.Run(src.ID); err != nil {
fmt.Printf("scraper %s error: %v\n", src.Name, err)
}
}
return nil
}

75
backend/internal/scraper/scraper.go Normal file
View File

@ -0,0 +1,75 @@
package scraper
import (
"context"
"time"
"github.com/tradarr/backend/internal/models"
)
type Article struct {
Title string
Content string
URL string
PublishedAt *time.Time
Symbols []string
}
type Scraper interface {
Name() string
Scrape(ctx context.Context, symbols []string) ([]Article, error)
}
// detectSymbols extrait les symboles mentionnés dans un texte
func DetectSymbols(text string, watchlist []string) []string {
found := map[string]bool{}
for _, s := range watchlist {
// Recherche du symbole en majuscules dans le texte
if containsWord(text, s) {
found[s] = true
}
}
result := make([]string, 0, len(found))
for s := range found {
result = append(result, s)
}
return result
}
func containsWord(text, word string) bool {
upper := []byte(text)
w := []byte(word)
for i := 0; i <= len(upper)-len(w); i++ {
match := true
for j := range w {
c := upper[i+j]
if c >= 'a' && c <= 'z' {
c -= 32
}
if c != w[j] {
match = false
break
}
}
if match {
// Vérifier que c'est un mot entier
before := i == 0 || !isAlphaNum(upper[i-1])
after := i+len(w) >= len(upper) || !isAlphaNum(upper[i+len(w)])
if before && after {
return true
}
}
}
return false
}
func isAlphaNum(b byte) bool {
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')
}
// ScraperResult est le résultat d'un job de scraping
type ScraperResult struct {
Source *models.Source
Articles []Article
Err error
}

128
backend/internal/scraper/stocktwits/stocktwits.go Normal file
View File

@ -0,0 +1,128 @@
package stocktwits
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/tradarr/backend/internal/scraper"
)
const apiBase = "https://api.stocktwits.com/api/2"
type StockTwits struct {
client *http.Client
}
func New() *StockTwits {
return &StockTwits{
client: &http.Client{Timeout: 15 * time.Second},
}
}
func (s *StockTwits) Name() string { return "stocktwits" }
type apiResponse struct {
Response struct {
Status int `json:"status"`
Error string `json:"error,omitempty"`
} `json:"response"`
Messages []struct {
ID int `json:"id"`
Body string `json:"body"`
CreatedAt string `json:"created_at"`
User struct {
Username string `json:"username"`
} `json:"user"`
Entities struct {
Sentiment *struct {
Basic string `json:"basic"`
} `json:"sentiment"`
} `json:"entities"`
} `json:"messages"`
}
func (s *StockTwits) Scrape(ctx context.Context, symbols []string) ([]scraper.Article, error) {
var articles []scraper.Article
for i, symbol := range symbols {
// Délai entre les requêtes pour éviter le rate limiting
if i > 0 {
select {
case <-ctx.Done():
return articles, ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
msgs, err := s.fetchSymbol(ctx, symbol)
if err != nil {
fmt.Printf("stocktwits %s: %v\n", symbol, err)
continue
}
articles = append(articles, msgs...)
}
return articles, nil
}
func (s *StockTwits) fetchSymbol(ctx context.Context, symbol string) ([]scraper.Article, error) {
url := fmt.Sprintf("%s/streams/symbol/%s.json", apiBase, symbol)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36")
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode == 429 {
return nil, fmt.Errorf("rate limited by StockTwits for %s", symbol)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("StockTwits returned HTTP %d for %s: %s", resp.StatusCode, symbol, string(body))
}
var data apiResponse
if err := json.Unmarshal(body, &data); err != nil {
return nil, fmt.Errorf("parse response for %s: %w", symbol, err)
}
// L'API StockTwits retourne un status dans le body même en HTTP 200
if data.Response.Status != 0 && data.Response.Status != 200 {
return nil, fmt.Errorf("StockTwits API error %d for %s: %s", data.Response.Status, symbol, data.Response.Error)
}
var articles []scraper.Article
for _, msg := range data.Messages {
if msg.Body == "" {
continue
}
sentiment := ""
if msg.Entities.Sentiment != nil {
sentiment = " [" + msg.Entities.Sentiment.Basic + "]"
}
title := fmt.Sprintf("$%s — @%s%s", symbol, msg.User.Username, sentiment)
publishedAt, _ := time.Parse(time.RFC3339, msg.CreatedAt)
msgURL := fmt.Sprintf("https://stocktwits.com/%s/message/%d", msg.User.Username, msg.ID)
articles = append(articles, scraper.Article{
Title: title,
Content: msg.Body,
URL: msgURL,
PublishedAt: &publishedAt,
Symbols: []string{symbol},
})
}
fmt.Printf("stocktwits %s: %d messages fetched\n", symbol, len(articles))
return articles, nil
}

126
backend/internal/scraper/yahoofinance/yahoofinance.go Normal file
View File

@ -0,0 +1,126 @@
package yahoofinance
import (
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/tradarr/backend/internal/scraper"
)
type YahooFinance struct {
client *http.Client
}
func New() *YahooFinance {
return &YahooFinance{
client: &http.Client{Timeout: 15 * time.Second},
}
}
func (y *YahooFinance) Name() string { return "stocktwits" } // garde le même type en DB
type rssFeed struct {
Channel struct {
Items []struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
} `xml:"item"`
} `xml:"channel"`
}
func (y *YahooFinance) Scrape(ctx context.Context, symbols []string) ([]scraper.Article, error) {
var articles []scraper.Article
for i, symbol := range symbols {
if i > 0 {
select {
case <-ctx.Done():
return articles, ctx.Err()
case <-time.After(300 * time.Millisecond):
}
}
items, err := y.fetchSymbol(ctx, symbol)
if err != nil {
fmt.Printf("yahoofinance %s: %v\n", symbol, err)
continue
}
articles = append(articles, items...)
fmt.Printf("yahoofinance %s: %d articles fetched\n", symbol, len(items))
}
return articles, nil
}
func (y *YahooFinance) fetchSymbol(ctx context.Context, symbol string) ([]scraper.Article, error) {
url := fmt.Sprintf(
"https://feeds.finance.yahoo.com/rss/2.0/headline?s=%s&region=US&lang=en-US",
symbol,
)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Tradarr/1.0)")
req.Header.Set("Accept", "application/rss+xml, application/xml, text/xml")
resp, err := y.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
var feed rssFeed
if err := xml.NewDecoder(resp.Body).Decode(&feed); err != nil {
return nil, fmt.Errorf("parse RSS: %w", err)
}
var articles []scraper.Article
for _, item := range feed.Channel.Items {
title := strings.TrimSpace(item.Title)
link := strings.TrimSpace(item.Link)
if title == "" || link == "" {
continue
}
var publishedAt *time.Time
if item.PubDate != "" {
formats := []string{
time.RFC1123Z,
time.RFC1123,
"Mon, 02 Jan 2006 15:04:05 -0700",
}
for _, f := range formats {
if t, err := time.Parse(f, item.PubDate); err == nil {
publishedAt = &t
break
}
}
}
content := strings.TrimSpace(item.Description)
if content == "" {
content = title
}
articles = append(articles, scraper.Article{
Title: title,
Content: content,
URL: link,
PublishedAt: publishedAt,
Symbols: []string{symbol},
})
}
return articles, nil
}