diff --git a/Dockerfile b/Dockerfile index 8fdbb72..34e4b38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN go mod download COPY . . # Build the application with optimizations -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-s -w" -o tercul ./cmd/api +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-s -w" -o tercul ./cmd/cli # Use a small alpine image for the final container FROM alpine:latest @@ -33,5 +33,5 @@ COPY --from=builder /app/tercul . # Expose the application port EXPOSE 8080 -# Command to run the application -CMD ["./tercul"] +# Command to run the API server +CMD ["./tercul", "serve"] diff --git a/README.md b/README.md index 4ac05ba..6414c83 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The Tercul backend is built using a Domain-Driven Design (DDD-lite) approach, em - **Command Query Responsibility Segregation (CQRS):** Application logic is separated into Commands (for writing data) and Queries (for reading data). This allows for optimized, scalable, and maintainable services. - **Clean Architecture:** Dependencies flow inwards, with inner layers (domain) having no knowledge of outer layers (infrastructure). -- **Dependency Injection:** Services and repositories are instantiated at the application's entry point (`cmd/api/main.go`) and injected as dependencies, promoting loose coupling and testability. +- **Dependency Injection:** Services and repositories are instantiated at the application's entry point (`cmd/cli`) and injected as dependencies, promoting loose coupling and testability. For a more detailed explanation of the architectural vision and ongoing refactoring efforts, please see `refactor.md`. @@ -55,10 +55,26 @@ The application will automatically connect to these services. For a full list of 2. **Run the API server:** ```bash - go run cmd/api/main.go + go run cmd/cli/main.go serve + ``` + Or build and run: + ```bash + go build -o bin/tercul ./cmd/cli + ./bin/tercul serve ``` The API server will be available at `http://localhost:8080`. The GraphQL playground can be accessed at `http://localhost:8080/playground`. +### Available Commands + +The Tercul CLI provides several commands: + +- `tercul serve` - Start the GraphQL API server +- `tercul worker` - Start background job workers +- `tercul enrich --type --id ` - Enrich entities with external data +- `tercul bleve-migrate --index ` - Migrate translations to Bleve index + +Run `tercul --help` for more information. + ## Running Tests To ensure code quality and correctness, run the full suite of linters and tests: diff --git a/cmd/cli/commands/bleve_migrate.go b/cmd/cli/commands/bleve_migrate.go new file mode 100644 index 0000000..6acb12b --- /dev/null +++ b/cmd/cli/commands/bleve_migrate.go @@ -0,0 +1,415 @@ +package commands + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/spf13/cobra" + "tercul/internal/data/sql" + "tercul/internal/domain" + "tercul/internal/platform/config" + "tercul/internal/platform/db" + "tercul/internal/platform/log" +) + +const ( + // Default batch size for processing translations + defaultBatchSize = 50000 + // Checkpoint file to track progress + checkpointFile = ".bleve_migration_checkpoint" +) + +type checkpoint struct { + LastProcessedID uint `json:"last_processed_id"` + TotalProcessed int `json:"total_processed"` + LastUpdated time.Time `json:"last_updated"` +} + +// NewBleveMigrateCommand creates a new Cobra command for Bleve migration +func NewBleveMigrateCommand() *cobra.Command { + var ( + indexPath string + batchSize int + resume bool + verify bool + ) + + cmd := &cobra.Command{ + Use: "bleve-migrate", + Short: "Migrate translations from PostgreSQL to Bleve index", + Long: `Migrate all translations from PostgreSQL database to a Bleve search index. + +This command: +- Fetches all translations from the database +- Indexes them in batches for efficient processing +- Supports resuming from checkpoints +- Provides progress tracking +- Can verify indexed data after migration + +Example: + tercul bleve-migrate --index ./data/bleve_index --batch 50000 --verify`, + RunE: func(cmd *cobra.Command, args []string) error { + if indexPath == "" { + return fmt.Errorf("index path is required (use --index flag)") + } + + // Initialize logger + log.Init("bleve-migrate", "development") + logger := log.FromContext(context.Background()) + + // Load configuration + cfg, err := config.LoadConfig() + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Initialize database + database, err := db.InitDB(cfg, nil) + if err != nil { + return fmt.Errorf("failed to initialize database: %w", err) + } + defer func() { + if err := db.Close(database); err != nil { + logger.Error(err, "Error closing database") + } + }() + + // Create repositories + repos := sql.NewRepositories(database, cfg) + + // Initialize or open Bleve index + logger.Info(fmt.Sprintf("Initializing Bleve index at %s", indexPath)) + index, err := initBleveIndex(indexPath) + if err != nil { + return fmt.Errorf("failed to initialize Bleve index: %w", err) + } + defer func() { + if err := index.Close(); err != nil { + logger.Error(err, "Error closing Bleve index") + } + }() + + // Load checkpoint if resuming + var cp *checkpoint + if resume { + cp = loadCheckpoint() + if cp != nil { + logger.Info(fmt.Sprintf("Resuming from checkpoint: last_id=%d, total_processed=%d", cp.LastProcessedID, cp.TotalProcessed)) + } + } + + // Run migration + ctx := context.Background() + stats, err := migrateTranslations(ctx, repos.Translation, index, batchSize, cp, logger, ctx) + if err != nil { + return fmt.Errorf("migration failed: %w", err) + } + + logger.Info(fmt.Sprintf("Migration completed: indexed=%d, errors=%d, duration=%v", stats.TotalIndexed, stats.TotalErrors, stats.Duration)) + + // Verify if requested + if verify { + logger.Info("Verifying indexed translations") + if err := verifyIndex(index, repos.Translation, logger, ctx); err != nil { + return fmt.Errorf("verification failed: %w", err) + } + logger.Info("Verification completed successfully") + } + + // Clean up checkpoint file + if err := os.Remove(checkpointFile); err != nil && !os.IsNotExist(err) { + logger.Warn(fmt.Sprintf("Failed to remove checkpoint file: %v", err)) + } + + return nil + }, + } + + // Add flags + cmd.Flags().StringVarP(&indexPath, "index", "i", "", "Path to Bleve index directory (required)") + cmd.Flags().IntVarP(&batchSize, "batch", "b", defaultBatchSize, "Batch size for processing translations") + cmd.Flags().BoolVarP(&resume, "resume", "r", false, "Resume from last checkpoint") + cmd.Flags().BoolVarP(&verify, "verify", "v", false, "Verify indexed translations after migration") + + // Mark index as required + _ = cmd.MarkFlagRequired("index") + + return cmd +} + +// initBleveIndex creates or opens a Bleve index with the appropriate mapping for translations +func initBleveIndex(indexPath string) (bleve.Index, error) { + // Check if index already exists + index, err := bleve.Open(indexPath) + if err == nil { + return index, nil + } + + // Index doesn't exist, create it + mapping := bleve.NewIndexMapping() + + // Create document mapping for Translation + translationMapping := bleve.NewDocumentMapping() + + // ID field (not analyzed, stored) + idMapping := bleve.NewTextFieldMapping() + idMapping.Store = true + idMapping.Index = true + idMapping.Analyzer = "keyword" + translationMapping.AddFieldMappingsAt("id", idMapping) + + // Title field (analyzed, stored) + titleMapping := bleve.NewTextFieldMapping() + titleMapping.Store = true + titleMapping.Index = true + titleMapping.Analyzer = "standard" + translationMapping.AddFieldMappingsAt("title", titleMapping) + + // Content field (analyzed, stored) + contentMapping := bleve.NewTextFieldMapping() + contentMapping.Store = true + contentMapping.Index = true + contentMapping.Analyzer = "standard" + translationMapping.AddFieldMappingsAt("content", contentMapping) + + // Description field (analyzed, stored) + descriptionMapping := bleve.NewTextFieldMapping() + descriptionMapping.Store = true + descriptionMapping.Index = true + descriptionMapping.Analyzer = "standard" + translationMapping.AddFieldMappingsAt("description", descriptionMapping) + + // Language field (not analyzed, stored, for filtering) + languageMapping := bleve.NewTextFieldMapping() + languageMapping.Store = true + languageMapping.Index = true + languageMapping.Analyzer = "keyword" + translationMapping.AddFieldMappingsAt("language", languageMapping) + + // Status field (not analyzed, stored, for filtering) + statusMapping := bleve.NewTextFieldMapping() + statusMapping.Store = true + statusMapping.Index = true + statusMapping.Analyzer = "keyword" + translationMapping.AddFieldMappingsAt("status", statusMapping) + + // TranslatableID field (not analyzed, stored) + translatableIDMapping := bleve.NewNumericFieldMapping() + translatableIDMapping.Store = true + translatableIDMapping.Index = true + translationMapping.AddFieldMappingsAt("translatable_id", translatableIDMapping) + + // TranslatableType field (not analyzed, stored, for filtering) + translatableTypeMapping := bleve.NewTextFieldMapping() + translatableTypeMapping.Store = true + translatableTypeMapping.Index = true + translatableTypeMapping.Analyzer = "keyword" + translationMapping.AddFieldMappingsAt("translatable_type", translatableTypeMapping) + + // TranslatorID field (not analyzed, stored) + translatorIDMapping := bleve.NewNumericFieldMapping() + translatorIDMapping.Store = true + translatorIDMapping.Index = true + translationMapping.AddFieldMappingsAt("translator_id", translatorIDMapping) + + // Add translation mapping to index + mapping.AddDocumentMapping("translation", translationMapping) + + // Create index directory if it doesn't exist + if err := os.MkdirAll(filepath.Dir(indexPath), 0755); err != nil { + return nil, fmt.Errorf("failed to create index directory: %w", err) + } + + // Create the index + index, err = bleve.New(indexPath, mapping) + if err != nil { + return nil, fmt.Errorf("failed to create Bleve index: %w", err) + } + + return index, nil +} + +type migrationStats struct { + TotalIndexed int + TotalErrors int + Duration time.Duration +} + +// migrateTranslations migrates all translations from PostgreSQL to Bleve index +func migrateTranslations( + ctx context.Context, + repo domain.TranslationRepository, + index bleve.Index, + batchSize int, + cp *checkpoint, + logger *log.Logger, + ctxForLog context.Context, +) (*migrationStats, error) { + startTime := time.Now() + stats := &migrationStats{} + + // Fetch all translations + logger.Info("Fetching all translations from database") + translations, err := repo.ListAll(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch translations: %w", err) + } + + totalTranslations := len(translations) + logger.Info(fmt.Sprintf("Found %d translations", totalTranslations)) + + // Filter translations if resuming from checkpoint + if cp != nil && cp.LastProcessedID > 0 { + filtered := make([]domain.Translation, 0, len(translations)) + for _, t := range translations { + if t.ID > cp.LastProcessedID { + filtered = append(filtered, t) + } + } + translations = filtered + stats.TotalIndexed = cp.TotalProcessed + logger.Info(fmt.Sprintf("Filtered translations: remaining=%d, already_processed=%d", len(translations), cp.TotalProcessed)) + } + + // Process translations in batches + batch := make([]domain.Translation, 0, batchSize) + lastProcessedID := uint(0) + + for i, translation := range translations { + batch = append(batch, translation) + lastProcessedID = translation.ID + + // Process batch when it reaches the batch size or at the end + if len(batch) >= batchSize || i == len(translations)-1 { + if err := indexBatch(index, batch, logger); err != nil { + logger.Error(err, fmt.Sprintf("Failed to index batch of size %d", len(batch))) + stats.TotalErrors += len(batch) + // Continue with next batch instead of failing completely + } else { + stats.TotalIndexed += len(batch) + } + + // Save checkpoint + cpData := checkpoint{ + LastProcessedID: lastProcessedID, + TotalProcessed: stats.TotalIndexed, + LastUpdated: time.Now(), + } + if err := saveCheckpoint(&cpData); err != nil { + logger.Warn(fmt.Sprintf("Failed to save checkpoint: %v", err)) + } + + // Log progress + progress := float64(stats.TotalIndexed) / float64(totalTranslations) * 100 + logger.Info(fmt.Sprintf("Migration progress: %d/%d (%.2f%%)", stats.TotalIndexed, totalTranslations, progress)) + + // Clear batch + batch = batch[:0] + } + } + + stats.Duration = time.Since(startTime) + return stats, nil +} + +// indexBatch indexes a batch of translations +func indexBatch(index bleve.Index, translations []domain.Translation, logger *log.Logger) error { + batch := index.NewBatch() + for _, t := range translations { + doc := map[string]interface{}{ + "id": strconv.FormatUint(uint64(t.ID), 10), + "title": t.Title, + "content": t.Content, + "description": t.Description, + "language": t.Language, + "status": string(t.Status), + "translatable_id": t.TranslatableID, + "translatable_type": t.TranslatableType, + } + + if t.TranslatorID != nil { + doc["translator_id"] = *t.TranslatorID + } + + docID := fmt.Sprintf("translation_%d", t.ID) + if err := batch.Index(docID, doc); err != nil { + return fmt.Errorf("failed to add document to batch: %w", err) + } + } + + if err := index.Batch(batch); err != nil { + return fmt.Errorf("failed to index batch: %w", err) + } + + return nil +} + +// verifyIndex verifies that all translations in the database are indexed in Bleve +func verifyIndex(index bleve.Index, repo domain.TranslationRepository, logger *log.Logger, ctx context.Context) error { + // Fetch all translations + translations, err := repo.ListAll(ctx) + if err != nil { + return fmt.Errorf("failed to fetch translations: %w", err) + } + + logger.Info(fmt.Sprintf("Verifying %d indexed translations", len(translations))) + + missing := 0 + for _, t := range translations { + docID := fmt.Sprintf("translation_%d", t.ID) + doc, err := index.Document(docID) + if err != nil { + logger.Warn(fmt.Sprintf("Translation %d not found in index: %v", t.ID, err)) + missing++ + continue + } + if doc == nil { + logger.Warn(fmt.Sprintf("Translation %d not found in index (nil document)", t.ID)) + missing++ + continue + } + } + + if missing > 0 { + return fmt.Errorf("verification failed: %d translations missing from index", missing) + } + + logger.Info("All translations verified in index") + return nil +} + +// saveCheckpoint saves the migration checkpoint to a file +func saveCheckpoint(cp *checkpoint) error { + data, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("failed to marshal checkpoint: %w", err) + } + + if err := os.WriteFile(checkpointFile, data, 0644); err != nil { + return fmt.Errorf("failed to write checkpoint file: %w", err) + } + + return nil +} + +// loadCheckpoint loads the migration checkpoint from a file +func loadCheckpoint() *checkpoint { + data, err := os.ReadFile(checkpointFile) + if err != nil { + return nil + } + + var cp checkpoint + if err := json.Unmarshal(data, &cp); err != nil { + return nil + } + + return &cp +} + diff --git a/cmd/cli/commands/enrich.go b/cmd/cli/commands/enrich.go new file mode 100644 index 0000000..5aa587c --- /dev/null +++ b/cmd/cli/commands/enrich.go @@ -0,0 +1,110 @@ +package commands + +import ( + "context" + "fmt" + "strconv" + + "tercul/cmd/cli/internal/bootstrap" + "tercul/internal/enrichment" + "tercul/internal/platform/config" + "tercul/internal/platform/db" + "tercul/internal/platform/log" + + "github.com/spf13/cobra" +) + +// NewEnrichCommand creates a new Cobra command for enriching entities +func NewEnrichCommand() *cobra.Command { + var ( + entityType string + entityID string + ) + + cmd := &cobra.Command{ + Use: "enrich", + Short: "Enrich an entity with external data", + Long: `Enrich an entity (e.g., author) with external data from sources like OpenLibrary. + +Example: + tercul enrich --type author --id 123`, + RunE: func(cmd *cobra.Command, args []string) error { + if entityType == "" || entityID == "" { + return fmt.Errorf("both --type and --id are required") + } + + entityIDUint, err := strconv.ParseUint(entityID, 10, 64) + if err != nil { + return fmt.Errorf("invalid entity ID: %w", err) + } + + // Load configuration + cfg, err := config.LoadConfig() + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Initialize logger + log.Init("enrich-tool", "development") + database, err := db.InitDB(cfg, nil) // No metrics needed for this tool + if err != nil { + log.Fatal(err, "Failed to initialize database") + } + defer func() { + if err := db.Close(database); err != nil { + log.Error(err, "Error closing database") + } + }() + + // Bootstrap dependencies + weaviateClient, err := bootstrap.NewWeaviateClient(cfg) + if err != nil { + return fmt.Errorf("failed to create weaviate client: %w", err) + } + + deps, err := bootstrap.Bootstrap(cfg, database, weaviateClient) + if err != nil { + return fmt.Errorf("failed to bootstrap: %w", err) + } + + enrichmentSvc := enrichment.NewService() + + // Fetch, enrich, and save the entity + ctx := context.Background() + log.Info(fmt.Sprintf("Enriching %s with ID %d", entityType, entityIDUint)) + + switch entityType { + case "author": + author, err := deps.Repos.Author.GetByID(ctx, uint(entityIDUint)) + if err != nil { + return fmt.Errorf("failed to get author: %w", err) + } + + if err := enrichmentSvc.EnrichAuthor(ctx, author); err != nil { + return fmt.Errorf("failed to enrich author: %w", err) + } + + if err := deps.Repos.Author.Update(ctx, author); err != nil { + return fmt.Errorf("failed to save enriched author: %w", err) + } + + log.Info("Successfully enriched and saved author") + + default: + return fmt.Errorf("unknown entity type: %s", entityType) + } + + return nil + }, + } + + // Add flags + cmd.Flags().StringVarP(&entityType, "type", "t", "", "The type of entity to enrich (e.g., 'author')") + cmd.Flags().StringVarP(&entityID, "id", "i", "", "The ID of the entity to enrich") + + // Mark flags as required + _ = cmd.MarkFlagRequired("type") + _ = cmd.MarkFlagRequired("id") + + return cmd +} diff --git a/cmd/cli/commands/serve.go b/cmd/cli/commands/serve.go new file mode 100644 index 0000000..f9d7cea --- /dev/null +++ b/cmd/cli/commands/serve.go @@ -0,0 +1,194 @@ +package commands + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "tercul/cmd/cli/internal/bootstrap" + "tercul/internal/adapters/graphql" + "tercul/internal/observability" + platform_auth "tercul/internal/platform/auth" + "tercul/internal/platform/config" + "tercul/internal/platform/db" + app_log "tercul/internal/platform/log" + + "github.com/99designs/gqlgen/graphql/handler" + "github.com/99designs/gqlgen/graphql/playground" + "github.com/pressly/goose/v3" + "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/cobra" + "github.com/weaviate/weaviate-go-client/v5/weaviate" + "gorm.io/gorm" +) + +// NewServeCommand creates a new Cobra command for serving the API +func NewServeCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "serve", + Short: "Start the Tercul API server", + Long: `Start the Tercul GraphQL API server with all endpoints including: +- GraphQL query endpoint (/query) +- GraphQL Playground (/playground) +- Prometheus metrics (/metrics)`, + RunE: func(cmd *cobra.Command, args []string) error { + // Load configuration + cfg, err := config.LoadConfig() + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Initialize logger + app_log.Init("tercul-api", cfg.Environment) + obsLogger := observability.NewLogger("tercul-api", cfg.Environment) + + // Initialize OpenTelemetry Tracer Provider + tp, err := observability.TracerProvider("tercul-api", cfg.Environment) + if err != nil { + app_log.Fatal(err, "Failed to initialize OpenTelemetry tracer") + } + defer func() { + if err := tp.Shutdown(context.Background()); err != nil { + app_log.Error(err, "Error shutting down tracer provider") + } + }() + + // Initialize Prometheus metrics + reg := prometheus.NewRegistry() + metrics := observability.NewMetrics(reg) + + app_log.Info(fmt.Sprintf("Starting Tercul application in %s environment, version 1.0.0", cfg.Environment)) + + // Initialize database connection + database, err := db.InitDB(cfg, metrics) + if err != nil { + app_log.Fatal(err, "Failed to initialize database") + } + defer func() { + if err := db.Close(database); err != nil { + app_log.Error(err, "Error closing database") + } + }() + + // Run migrations + if err := runMigrations(database, cfg.MigrationPath); err != nil { + app_log.Fatal(err, "Failed to apply database migrations") + } + + // Initialize Weaviate client + weaviateCfg := weaviate.Config{ + Host: cfg.WeaviateHost, + Scheme: cfg.WeaviateScheme, + } + weaviateClient, err := weaviate.NewClient(weaviateCfg) + if err != nil { + app_log.Fatal(err, "Failed to create weaviate client") + } + + // Bootstrap application dependencies + deps, err := bootstrap.BootstrapWithMetrics(cfg, database, weaviateClient) + if err != nil { + return fmt.Errorf("failed to bootstrap application: %w", err) + } + + // Create GraphQL server + resolver := &graphql.Resolver{ + App: deps.Application, + } + + // Create the API server + apiHandler := newAPIServer(resolver, deps.JWTManager, metrics, obsLogger, reg) + + // Create the main HTTP server + mainServer := &http.Server{ + Addr: cfg.ServerPort, + Handler: apiHandler, + } + app_log.Info(fmt.Sprintf("API server listening on port %s", cfg.ServerPort)) + + // Start the main server in a goroutine + go func() { + if err := mainServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + app_log.Fatal(err, "Failed to start server") + } + }() + + // Wait for interrupt signal to gracefully shutdown the server + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + app_log.Info("Shutting down server...") + + // Graceful shutdown + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := mainServer.Shutdown(ctx); err != nil { + app_log.Error(err, "Server forced to shutdown") + } + + app_log.Info("Server shut down successfully") + return nil + }, + } + + return cmd +} + +// runMigrations applies database migrations using goose. +func runMigrations(gormDB *gorm.DB, migrationPath string) error { + sqlDB, err := gormDB.DB() + if err != nil { + return err + } + + if err := goose.SetDialect("postgres"); err != nil { + return err + } + + app_log.Info(fmt.Sprintf("Applying database migrations from %s", migrationPath)) + if err := goose.Up(sqlDB, migrationPath); err != nil { + return err + } + app_log.Info("Database migrations applied successfully") + return nil +} + +// newAPIServer creates a new http.ServeMux and configures it with all the API routes +func newAPIServer( + resolver *graphql.Resolver, + jwtManager *platform_auth.JWTManager, + metrics *observability.Metrics, + logger *observability.Logger, + reg *prometheus.Registry, +) *http.ServeMux { + // Configure the GraphQL server + c := graphql.Config{Resolvers: resolver} + c.Directives.Binding = graphql.Binding + + // Create the core GraphQL handler + graphqlHandler := handler.New(graphql.NewExecutableSchema(c)) + graphqlHandler.SetErrorPresenter(graphql.NewErrorPresenter()) + + // Create the middleware chain for the GraphQL endpoint. + // Middlewares are applied from bottom to top. + var chain http.Handler + chain = graphqlHandler + chain = metrics.PrometheusMiddleware(chain) + chain = observability.LoggingMiddleware(logger)(chain) // Must run after auth and tracing + chain = platform_auth.GraphQLAuthMiddleware(jwtManager)(chain) + chain = observability.TracingMiddleware(chain) + chain = observability.RequestIDMiddleware(chain) + + // Create a new ServeMux and register all handlers + mux := http.NewServeMux() + mux.Handle("/query", chain) + mux.Handle("/playground", playground.Handler("GraphQL Playground", "/query")) + mux.Handle("/metrics", observability.PrometheusHandler(reg)) + + return mux +} diff --git a/cmd/cli/commands/worker.go b/cmd/cli/commands/worker.go new file mode 100644 index 0000000..9714d7b --- /dev/null +++ b/cmd/cli/commands/worker.go @@ -0,0 +1,110 @@ +package commands + +import ( + "os" + "os/signal" + "syscall" + + "tercul/cmd/cli/internal/bootstrap" + "tercul/internal/jobs/sync" + "tercul/internal/platform/config" + "tercul/internal/platform/db" + app_log "tercul/internal/platform/log" + + "github.com/hibiken/asynq" + "github.com/spf13/cobra" +) + +// NewWorkerCommand creates a new Cobra command for running background workers +func NewWorkerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "worker", + Short: "Start the Tercul background worker", + Long: `Start the Tercul background worker to process async jobs including: +- Sync jobs (Weaviate indexing, etc.) +- Linguistic analysis jobs +- Trending calculation jobs`, + RunE: func(cmd *cobra.Command, args []string) error { + // Load configuration + cfg, err := config.LoadConfig() + if err != nil { + return err + } + + // Initialize logger + app_log.Init("tercul-worker", cfg.Environment) + app_log.Info("Starting Tercul worker...") + + // Initialize database connection + database, err := db.InitDB(cfg, nil) // No metrics needed for the worker + if err != nil { + app_log.Fatal(err, "Failed to initialize database") + } + defer func() { + if err := db.Close(database); err != nil { + app_log.Error(err, "Error closing database") + } + }() + + // Initialize Weaviate client + weaviateClient, err := bootstrap.NewWeaviateClient(cfg) + if err != nil { + app_log.Fatal(err, "Failed to create weaviate client") + } + + // Initialize Asynq client and server + redisConnection := asynq.RedisClientOpt{Addr: cfg.RedisAddr} + asynqClient := asynq.NewClient(redisConnection) + defer func() { + if err := asynqClient.Close(); err != nil { + app_log.Error(err, "Error closing asynq client") + } + }() + + srv := asynq.NewServer( + redisConnection, + asynq.Config{ + Concurrency: 10, // Example concurrency + Queues: map[string]int{ + "critical": 6, + "default": 3, + "low": 1, + }, + }, + ) + + // Create SyncJob with all dependencies + syncJob := sync.NewSyncJob(database, asynqClient, cfg, weaviateClient) + + // Create a new ServeMux for routing jobs + mux := asynq.NewServeMux() + + // Register all job handlers + sync.RegisterQueueHandlers(mux, syncJob) + // Placeholder for other job handlers that might be added in the future + // linguistics.RegisterLinguisticHandlers(mux, linguisticJob) + // trending.RegisterTrendingHandlers(mux, analyticsService) + + // Start the server in a goroutine + go func() { + if err := srv.Run(mux); err != nil { + app_log.Fatal(err, "Could not run asynq server") + } + }() + + app_log.Info("Worker started successfully.") + + // Wait for interrupt signal to gracefully shutdown the server + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + app_log.Info("Shutting down worker...") + srv.Shutdown() + app_log.Info("Worker shut down successfully.") + return nil + }, + } + + return cmd +} diff --git a/cmd/cli/internal/bootstrap/bootstrap.go b/cmd/cli/internal/bootstrap/bootstrap.go new file mode 100644 index 0000000..983a766 --- /dev/null +++ b/cmd/cli/internal/bootstrap/bootstrap.go @@ -0,0 +1,132 @@ +package bootstrap + +import ( + "tercul/internal/app" + "tercul/internal/app/analytics" + "tercul/internal/app/auth" + "tercul/internal/app/author" + "tercul/internal/app/authz" + "tercul/internal/app/book" + "tercul/internal/app/bookmark" + "tercul/internal/app/category" + "tercul/internal/app/collection" + "tercul/internal/app/comment" + "tercul/internal/app/contribution" + "tercul/internal/app/like" + "tercul/internal/app/localization" + appsearch "tercul/internal/app/search" + "tercul/internal/app/tag" + "tercul/internal/app/translation" + "tercul/internal/app/user" + "tercul/internal/app/work" + dbsql "tercul/internal/data/sql" + "tercul/internal/jobs/linguistics" + platform_auth "tercul/internal/platform/auth" + "tercul/internal/platform/config" + "tercul/internal/platform/search" + domainsearch "tercul/internal/domain/search" + + "gorm.io/gorm" + "github.com/weaviate/weaviate-go-client/v5/weaviate" +) + +// NewWeaviateClient creates a new Weaviate client from config +func NewWeaviateClient(cfg *config.Config) (*weaviate.Client, error) { + weaviateCfg := weaviate.Config{ + Host: cfg.WeaviateHost, + Scheme: cfg.WeaviateScheme, + } + return weaviate.NewClient(weaviateCfg) +} + +// Dependencies holds all initialized dependencies +type Dependencies struct { + Config *config.Config + Database *gorm.DB + WeaviateClient *weaviate.Client + SearchClient domainsearch.SearchClient + Repos *dbsql.Repositories + Application *app.Application + JWTManager *platform_auth.JWTManager + AnalysisRepo *linguistics.GORMAnalysisRepository + SentimentProvider *linguistics.GoVADERSentimentProvider +} + +// Bootstrap initializes all application dependencies +func Bootstrap(cfg *config.Config, database *gorm.DB, weaviateClient *weaviate.Client) (*Dependencies, error) { + // Create search client + searchClient := search.NewWeaviateWrapper(weaviateClient, cfg.WeaviateHost, cfg.SearchAlpha) + + // Create repositories + repos := dbsql.NewRepositories(database, cfg) + + // Create linguistics dependencies + analysisRepo := linguistics.NewGORMAnalysisRepository(database) + sentimentProvider, err := linguistics.NewGoVADERSentimentProvider() + if err != nil { + return nil, err + } + + // Create platform components + jwtManager := platform_auth.NewJWTManager(cfg) + + // Create application services + analyticsService := analytics.NewService(repos.Analytics, analysisRepo, repos.Translation, repos.Work, sentimentProvider) + localizationService := localization.NewService(repos.Localization) + searchService := appsearch.NewService(searchClient, localizationService) + authzService := authz.NewService(repos.Work, repos.Author, repos.User, repos.Translation) + authorService := author.NewService(repos.Author) + bookService := book.NewService(repos.Book, authzService) + bookmarkService := bookmark.NewService(repos.Bookmark, analyticsService) + categoryService := category.NewService(repos.Category) + collectionService := collection.NewService(repos.Collection) + commentService := comment.NewService(repos.Comment, authzService, analyticsService) + contributionCommands := contribution.NewCommands(repos.Contribution, authzService) + contributionService := contribution.NewService(contributionCommands) + likeService := like.NewService(repos.Like, analyticsService) + tagService := tag.NewService(repos.Tag) + translationService := translation.NewService(repos.Translation, authzService) + userService := user.NewService(repos.User, authzService, repos.UserProfile) + authService := auth.NewService(repos.User, jwtManager) + workService := work.NewService(repos.Work, repos.Author, repos.User, searchClient, authzService, analyticsService) + + // Create application + application := app.NewApplication( + authorService, + bookService, + bookmarkService, + categoryService, + collectionService, + commentService, + contributionService, + likeService, + tagService, + translationService, + userService, + localizationService, + authService, + authzService, + workService, + searchService, + analyticsService, + ) + + return &Dependencies{ + Config: cfg, + Database: database, + WeaviateClient: weaviateClient, + SearchClient: searchClient, + Repos: repos, + Application: application, + JWTManager: jwtManager, + AnalysisRepo: analysisRepo, + SentimentProvider: sentimentProvider, + }, nil +} + +// BootstrapWithMetrics initializes dependencies with metrics support +func BootstrapWithMetrics(cfg *config.Config, database *gorm.DB, weaviateClient *weaviate.Client) (*Dependencies, error) { + // For now, same as Bootstrap, but can be extended if metrics are needed in bootstrap + return Bootstrap(cfg, database, weaviateClient) +} + diff --git a/cmd/cli/main.go b/cmd/cli/main.go new file mode 100644 index 0000000..45aabe4 --- /dev/null +++ b/cmd/cli/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "os" + + "tercul/cmd/cli/commands" + + "github.com/spf13/cobra" +) + +func main() { + rootCmd := &cobra.Command{ + Use: "tercul", + Short: "Tercul CLI - Command-line tools for the Tercul backend", + Long: `Tercul CLI provides various command-line tools for managing and operating +the Tercul backend, including data migration, indexing, and maintenance tasks.`, + } + + // Add subcommands + rootCmd.AddCommand(commands.NewServeCommand()) + rootCmd.AddCommand(commands.NewWorkerCommand()) + rootCmd.AddCommand(commands.NewEnrichCommand()) + rootCmd.AddCommand(commands.NewBleveMigrateCommand()) + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index c556d19..82153ff 100644 --- a/go.mod +++ b/go.mod @@ -42,10 +42,30 @@ require ( github.com/ClickHouse/ch-go v0.67.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.40.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/blevesearch/bleve/v2 v2.5.5 // indirect + github.com/blevesearch/bleve_index_api v1.2.11 // indirect + github.com/blevesearch/geo v0.2.4 // indirect + github.com/blevesearch/go-faiss v1.0.26 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.3.13 // indirect + github.com/blevesearch/segment v0.9.1 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.1.0 // indirect + github.com/blevesearch/zapx/v11 v11.4.2 // indirect + github.com/blevesearch/zapx/v12 v12.4.2 // indirect + github.com/blevesearch/zapx/v13 v13.4.2 // indirect + github.com/blevesearch/zapx/v14 v14.4.2 // indirect + github.com/blevesearch/zapx/v15 v15.4.2 // indirect + github.com/blevesearch/zapx/v16 v16.2.7 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coder/websocket v1.8.12 // indirect @@ -89,7 +109,9 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.7.5 // indirect @@ -99,6 +121,7 @@ require ( github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect @@ -117,7 +140,10 @@ require ( github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -146,6 +172,7 @@ require ( github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.1 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/subosito/gotenv v1.6.0 // indirect @@ -159,6 +186,7 @@ require ( github.com/ydb-platform/ydb-go-sdk/v3 v3.108.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/ziutek/mymysql v1.5.4 // indirect + go.etcd.io/bbolt v1.4.3 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect diff --git a/go.sum b/go.sum index fa96c28..26c66b7 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiU github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= +github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -55,6 +57,45 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= +github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blevesearch/bleve/v2 v2.5.5 h1:lzC89QUCco+y1qBnJxGqm4AbtsdsnlUvq0kXok8n3C8= +github.com/blevesearch/bleve/v2 v2.5.5/go.mod h1:t5WoESS5TDteTdnjhhvpA1BpLYErOBX2IQViTMLK7wo= +github.com/blevesearch/bleve_index_api v1.2.11 h1:bXQ54kVuwP8hdrXUSOnvTQfgK0KI1+f9A0ITJT8tX1s= +github.com/blevesearch/bleve_index_api v1.2.11/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0= +github.com/blevesearch/geo v0.2.4 h1:ECIGQhw+QALCZaDcogRTNSJYQXRtC8/m8IKiA706cqk= +github.com/blevesearch/geo v0.2.4/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= +github.com/blevesearch/go-faiss v1.0.26 h1:4dRLolFgjPyjkaXwff4NfbZFdE/dfywbzDqporeQvXI= +github.com/blevesearch/go-faiss v1.0.26/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.3.13 h1:ZPjv/4VwWvHJZKeMSgScCapOy8+DdmsmRyLmSB88UoY= +github.com/blevesearch/scorch_segment_api/v2 v2.3.13/go.mod h1:ENk2LClTehOuMS8XzN3UxBEErYmtwkE7MAArFTXs9Vc= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= +github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= +github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= +github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= +github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= +github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= +github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= +github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= +github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= +github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= +github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= +github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= +github.com/blevesearch/zapx/v16 v16.2.7 h1:xcgFRa7f/tQXOwApVq7JWgPYSlzyUMmkuYa54tMDuR0= +github.com/blevesearch/zapx/v16 v16.2.7/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -85,6 +126,7 @@ github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7np github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -252,6 +294,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -263,6 +307,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -280,6 +325,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/hibiken/asynq v0.25.1 h1:phj028N0nm15n8O2ims+IvJ2gz4k2auvermngh9JhTw= github.com/hibiken/asynq v0.25.1/go.mod h1:pazWNOLBu0FEynQRBvHA26qdIKRSmfdIfUm4HdsLmXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -303,6 +350,8 @@ github.com/jonreiter/govader v0.0.0-20250429093935-f6505c8d03cc h1:Zvn/U2151AlhF github.com/jonreiter/govader v0.0.0-20250429093935-f6505c8d03cc/go.mod h1:1o8G6XiwYAsUAF/bTOC5BAXjSNFzJD/RE9uQyssNwac= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= @@ -370,9 +419,16 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -470,7 +526,10 @@ github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8 github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= @@ -527,6 +586,8 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -740,6 +801,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314=