From f434b26dd4152b1953d01981527ffb72d53a18aa Mon Sep 17 00:00:00 2001 From: Damir Mukimov Date: Fri, 26 Dec 2025 13:18:00 +0100 Subject: [PATCH] Enhance configuration management and testing for backend - Update .gitignore to selectively ignore pkg/ directories at the root level - Modify CI workflow to verify all Go packages can be listed - Introduce configuration management with a new config package, including loading environment variables - Add comprehensive tests for configuration loading and environment variable handling - Implement Neo4j database interaction functions with corresponding tests for data extraction --- .gitea/workflows/ci.yml | 3 +- .gitignore | 5 +- bugulma/backend/pkg/config/config.go | 80 ++++++++ bugulma/backend/pkg/config/config_test.go | 104 +++++++++++ bugulma/backend/pkg/database/neo4j.go | 205 +++++++++++++++++++++ bugulma/backend/pkg/database/neo4j_test.go | 154 ++++++++++++++++ 6 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 bugulma/backend/pkg/config/config.go create mode 100644 bugulma/backend/pkg/config/config_test.go create mode 100644 bugulma/backend/pkg/database/neo4j.go create mode 100644 bugulma/backend/pkg/database/neo4j_test.go diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 88fa338..defb6a2 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -87,7 +87,8 @@ jobs: pwd ls -la go.mod go list -m - go list ./pkg/config + # Verify all packages can be listed + go list ./... env: GO111MODULE: on GOPROXY: https://proxy.golang.org,direct diff --git a/.gitignore b/.gitignore index 75a75ad..0e5d12a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,10 @@ go.work # Build output bin/ dist/ -pkg/ +# Note: pkg/ is used for source code in bugulma/backend, so we only ignore build artifact pkg/ directories +# Ignore pkg/ only at root level or in specific build contexts, not in source code directories +/pkg/ +*.a # Vendor directory vendor/ diff --git a/bugulma/backend/pkg/config/config.go b/bugulma/backend/pkg/config/config.go new file mode 100644 index 0000000..75d7b54 --- /dev/null +++ b/bugulma/backend/pkg/config/config.go @@ -0,0 +1,80 @@ +package config + +import "os" + +type Config struct { + ServerPort string + JWTSecret string + CORSOrigin string + + // PostgreSQL configuration + PostgresHost string + PostgresPort string + PostgresUser string + PostgresPassword string + PostgresDB string + PostgresSSLMode string + + // Neo4j configuration + Neo4jURI string + Neo4jUsername string + Neo4jPassword string + Neo4jDatabase string + Neo4jEnabled bool + + // Redis configuration + RedisURL string + + // Ollama configuration + OllamaURL string + OllamaModel string + OllamaUsername string + OllamaPassword string + + // Google Maps API configuration + GoogleMapsAPIKey string + GoogleCloudProjectID string +} + +func Load() *Config { + return &Config{ + ServerPort: getEnv("SERVER_PORT", "8080"), + JWTSecret: getEnv("JWT_SECRET", "your-secret-key-change-in-production"), + CORSOrigin: getEnv("CORS_ORIGIN", "http://localhost:3000"), + + // PostgreSQL defaults + PostgresHost: getEnv("POSTGRES_HOST", "localhost"), + PostgresPort: getEnv("POSTGRES_PORT", "5432"), + PostgresUser: getEnv("POSTGRES_USER", "bugulma"), + PostgresPassword: getEnv("POSTGRES_PASSWORD", "bugulma"), + PostgresDB: getEnv("POSTGRES_DB", "bugulma_city"), + PostgresSSLMode: getEnv("POSTGRES_SSLMODE", "disable"), + + // Neo4j defaults (disabled by default) + Neo4jURI: getEnv("NEO4J_URI", "neo4j://localhost:7687"), + Neo4jUsername: getEnv("NEO4J_USERNAME", "neo4j"), + Neo4jPassword: getEnv("NEO4J_PASSWORD", "password"), + Neo4jDatabase: getEnv("NEO4J_DATABASE", "neo4j"), + Neo4jEnabled: getEnv("NEO4J_ENABLED", "false") == "true", + + // Redis defaults + RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"), + + // Ollama defaults + OllamaURL: getEnv("OLLAMA_URL", "http://localhost:11434"), + OllamaModel: getEnv("OLLAMA_MODEL", "qwen2.5:7b"), + OllamaUsername: getEnv("OLLAMA_USERNAME", ""), + OllamaPassword: getEnv("OLLAMA_PASSWORD", ""), + + // Google Maps API defaults + GoogleMapsAPIKey: getEnv("GOOGLE_KG_API_KEY", ""), // Reuse KG API key if it works for Geocoding + GoogleCloudProjectID: getEnv("GOOGLE_CLOUD_PROJECT_ID", ""), + } +} + +func getEnv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} diff --git a/bugulma/backend/pkg/config/config_test.go b/bugulma/backend/pkg/config/config_test.go new file mode 100644 index 0000000..76692e9 --- /dev/null +++ b/bugulma/backend/pkg/config/config_test.go @@ -0,0 +1,104 @@ +package config + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetEnv(t *testing.T) { + // Test when environment variable is set + os.Setenv("TEST_VAR", "test_value") + defer os.Unsetenv("TEST_VAR") + + result := getEnv("TEST_VAR", "fallback") + assert.Equal(t, "test_value", result) + + // Test when environment variable is not set + result = getEnv("NON_EXISTENT_VAR", "fallback") + assert.Equal(t, "fallback", result) +} + +func TestLoad(t *testing.T) { + // Clear any existing test environment variables + testVars := []string{ + "SERVER_PORT", "JWT_SECRET", "CORS_ORIGIN", + "POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_DB", "POSTGRES_SSLMODE", + "NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "NEO4J_DATABASE", "NEO4J_ENABLED", + } + + for _, v := range testVars { + os.Unsetenv(v) + } + + // Test with all defaults + config := Load() + + assert.Equal(t, "8080", config.ServerPort) + assert.Equal(t, "your-secret-key-change-in-production", config.JWTSecret) + assert.Equal(t, "http://localhost:3000", config.CORSOrigin) + + assert.Equal(t, "localhost", config.PostgresHost) + assert.Equal(t, "5432", config.PostgresPort) + assert.Equal(t, "bugulma", config.PostgresUser) + assert.Equal(t, "bugulma", config.PostgresPassword) + assert.Equal(t, "bugulma_city", config.PostgresDB) + assert.Equal(t, "disable", config.PostgresSSLMode) + + assert.Equal(t, "neo4j://localhost:7687", config.Neo4jURI) + assert.Equal(t, "neo4j", config.Neo4jUsername) + assert.Equal(t, "password", config.Neo4jPassword) + assert.Equal(t, "neo4j", config.Neo4jDatabase) + assert.False(t, config.Neo4jEnabled) +} + +func TestLoad_WithEnvironmentVariables(t *testing.T) { + // Set test environment variables + os.Setenv("SERVER_PORT", "9090") + os.Setenv("JWT_SECRET", "test-secret") + os.Setenv("CORS_ORIGIN", "http://test.com") + os.Setenv("POSTGRES_HOST", "test-host") + os.Setenv("POSTGRES_PORT", "9999") + os.Setenv("POSTGRES_USER", "test-user") + os.Setenv("POSTGRES_PASSWORD", "test-pass") + os.Setenv("POSTGRES_DB", "test-db") + os.Setenv("POSTGRES_SSLMODE", "require") + os.Setenv("NEO4J_URI", "neo4j://test:7688") + os.Setenv("NEO4J_USERNAME", "test-user") + os.Setenv("NEO4J_PASSWORD", "test-pass") + os.Setenv("NEO4J_DATABASE", "test-db") + os.Setenv("NEO4J_ENABLED", "true") + + defer func() { + // Clean up + testVars := []string{ + "SERVER_PORT", "JWT_SECRET", "CORS_ORIGIN", + "POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_DB", "POSTGRES_SSLMODE", + "NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "NEO4J_DATABASE", "NEO4J_ENABLED", + } + for _, v := range testVars { + os.Unsetenv(v) + } + }() + + // Test with environment variables set + config := Load() + + assert.Equal(t, "9090", config.ServerPort) + assert.Equal(t, "test-secret", config.JWTSecret) + assert.Equal(t, "http://test.com", config.CORSOrigin) + + assert.Equal(t, "test-host", config.PostgresHost) + assert.Equal(t, "9999", config.PostgresPort) + assert.Equal(t, "test-user", config.PostgresUser) + assert.Equal(t, "test-pass", config.PostgresPassword) + assert.Equal(t, "test-db", config.PostgresDB) + assert.Equal(t, "require", config.PostgresSSLMode) + + assert.Equal(t, "neo4j://test:7688", config.Neo4jURI) + assert.Equal(t, "test-user", config.Neo4jUsername) + assert.Equal(t, "test-pass", config.Neo4jPassword) + assert.Equal(t, "test-db", config.Neo4jDatabase) + assert.True(t, config.Neo4jEnabled) +} diff --git a/bugulma/backend/pkg/database/neo4j.go b/bugulma/backend/pkg/database/neo4j.go new file mode 100644 index 0000000..e7b47ee --- /dev/null +++ b/bugulma/backend/pkg/database/neo4j.go @@ -0,0 +1,205 @@ +package database + +import ( + "context" + "fmt" + "time" + + "github.com/neo4j/neo4j-go-driver/v5/neo4j" +) + +// Neo4jConfig holds configuration for Neo4j connection +type Neo4jConfig struct { + URI string + Username string + Password string + Database string +} + +// NewNeo4jDriver creates a new Neo4j driver instance +func NewNeo4jDriver(config Neo4jConfig) (neo4j.DriverWithContext, error) { + driver, err := neo4j.NewDriverWithContext( + config.URI, + neo4j.BasicAuth(config.Username, config.Password, ""), + func(c *neo4j.Config) { + c.MaxConnectionPoolSize = 50 + c.ConnectionAcquisitionTimeout = 30 * time.Second + c.MaxTransactionRetryTime = 30 * time.Second + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to create Neo4j driver: %w", err) + } + + // Verify connectivity + ctx := context.Background() + if err := driver.VerifyConnectivity(ctx); err != nil { + driver.Close(ctx) + return nil, fmt.Errorf("failed to verify Neo4j connectivity: %w", err) + } + + return driver, nil +} + +// InitializeSchema creates constraints and indexes in Neo4j +func InitializeSchema(ctx context.Context, driver neo4j.DriverWithContext, database string) error { + session := driver.NewSession(ctx, neo4j.SessionConfig{ + AccessMode: neo4j.AccessModeWrite, + DatabaseName: database, + }) + defer session.Close(ctx) + + // Constraints for uniqueness + constraints := []string{ + "CREATE CONSTRAINT business_id_unique IF NOT EXISTS FOR (b:Business) REQUIRE b.id IS UNIQUE", + "CREATE CONSTRAINT organization_id_unique IF NOT EXISTS FOR (o:Organization) REQUIRE o.id IS UNIQUE", + "CREATE CONSTRAINT site_id_unique IF NOT EXISTS FOR (s:Site) REQUIRE s.id IS UNIQUE", + "CREATE CONSTRAINT resource_flow_id_unique IF NOT EXISTS FOR (rf:ResourceFlow) REQUIRE rf.id IS UNIQUE", + "CREATE CONSTRAINT match_id_unique IF NOT EXISTS FOR (m:Match) REQUIRE m.id IS UNIQUE", + "CREATE CONSTRAINT shared_asset_id_unique IF NOT EXISTS FOR (sa:SharedAsset) REQUIRE sa.id IS UNIQUE", + } + + // Indexes for performance + indexes := []string{ + "CREATE INDEX organization_name_index IF NOT EXISTS FOR (o:Organization) ON (o.name)", + "CREATE INDEX organization_sector_index IF NOT EXISTS FOR (o:Organization) ON (o.sector)", + "CREATE INDEX organization_subtype_index IF NOT EXISTS FOR (o:Organization) ON (o.subtype)", + "CREATE INDEX site_location_index IF NOT EXISTS FOR (s:Site) ON (s.latitude, s.longitude)", + "CREATE INDEX site_type_index IF NOT EXISTS FOR (s:Site) ON (s.site_type)", + "CREATE INDEX resource_flow_type_direction_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.type, rf.direction)", + "CREATE INDEX resource_flow_type_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.type)", + "CREATE INDEX resource_flow_direction_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.direction)", + "CREATE INDEX match_status_index IF NOT EXISTS FOR (m:Match) ON (m.status)", + "CREATE INDEX match_score_index IF NOT EXISTS FOR (m:Match) ON (m.compatibility_score)", + "CREATE INDEX shared_asset_type_index IF NOT EXISTS FOR (sa:SharedAsset) ON (sa.type)", + } + + // Execute constraints + for _, constraint := range constraints { + if _, err := session.Run(ctx, constraint, nil); err != nil { + return fmt.Errorf("failed to create constraint: %w", err) + } + } + + // Execute indexes + for _, index := range indexes { + if _, err := session.Run(ctx, index, nil); err != nil { + return fmt.Errorf("failed to create index: %w", err) + } + } + + return nil +} + +// Helper functions for extracting values from Neo4j records + +// GetString extracts a string value from a map +func GetString(m map[string]interface{}, key string) string { + if val, ok := m[key]; ok { + if str, ok := val.(string); ok { + return str + } + } + return "" +} + +// GetFloat64 extracts a float64 value from a map +func GetFloat64(m map[string]interface{}, key string) float64 { + if val, ok := m[key]; ok { + switch v := val.(type) { + case float64: + return v + case int64: + return float64(v) + case int: + return float64(v) + } + } + return 0 +} + +// GetInt extracts an int value from a map +func GetInt(m map[string]interface{}, key string) int { + if val, ok := m[key]; ok { + switch v := val.(type) { + case int64: + return int(v) + case int: + return v + case float64: + return int(v) + } + } + return 0 +} + +// GetBool extracts a bool value from a map +func GetBool(m map[string]interface{}, key string) bool { + if val, ok := m[key]; ok { + if b, ok := val.(bool); ok { + return b + } + } + return false +} + +// GetTime extracts a time.Time value from a map +func GetTime(m map[string]interface{}, key string) time.Time { + if val, ok := m[key]; ok { + if t, ok := val.(time.Time); ok { + return t + } + if str, ok := val.(string); ok { + if parsed, err := time.Parse(time.RFC3339, str); err == nil { + return parsed + } + } + } + return time.Time{} +} + +// GetStringFromRecord extracts a string value from a Neo4j record +func GetStringFromRecord(record neo4j.Record, key string) string { + val, ok := record.Get(key) + if !ok { + return "" + } + if str, ok := val.(string); ok { + return str + } + return "" +} + +// GetFloat64FromRecord extracts a float64 value from a Neo4j record +func GetFloat64FromRecord(record neo4j.Record, key string) float64 { + val, ok := record.Get(key) + if !ok { + return 0 + } + switch v := val.(type) { + case float64: + return v + case int64: + return float64(v) + case int: + return float64(v) + } + return 0 +} + +// GetIntFromRecord extracts an int value from a Neo4j record +func GetIntFromRecord(record neo4j.Record, key string) int { + val, ok := record.Get(key) + if !ok { + return 0 + } + switch v := val.(type) { + case int64: + return int(v) + case int: + return v + case float64: + return int(v) + } + return 0 +} diff --git a/bugulma/backend/pkg/database/neo4j_test.go b/bugulma/backend/pkg/database/neo4j_test.go new file mode 100644 index 0000000..c68a1fa --- /dev/null +++ b/bugulma/backend/pkg/database/neo4j_test.go @@ -0,0 +1,154 @@ +package database + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetString(t *testing.T) { + t.Run("existing string key", func(t *testing.T) { + m := map[string]interface{}{"key": "value"} + result := GetString(m, "key") + assert.Equal(t, "value", result) + }) + + t.Run("non-existing key", func(t *testing.T) { + m := map[string]interface{}{} + result := GetString(m, "key") + assert.Equal(t, "", result) + }) + + t.Run("non-string value", func(t *testing.T) { + m := map[string]interface{}{"key": 123} + result := GetString(m, "key") + assert.Equal(t, "", result) + }) +} + +func TestGetFloat64(t *testing.T) { + t.Run("float64 value", func(t *testing.T) { + m := map[string]interface{}{"key": 3.14} + result := GetFloat64(m, "key") + assert.Equal(t, 3.14, result) + }) + + t.Run("int64 value", func(t *testing.T) { + m := map[string]interface{}{"key": int64(42)} + result := GetFloat64(m, "key") + assert.Equal(t, 42.0, result) + }) + + t.Run("int value", func(t *testing.T) { + m := map[string]interface{}{"key": 42} + result := GetFloat64(m, "key") + assert.Equal(t, 42.0, result) + }) + + t.Run("non-existing key", func(t *testing.T) { + m := map[string]interface{}{} + result := GetFloat64(m, "key") + assert.Equal(t, 0.0, result) + }) + + t.Run("non-numeric value", func(t *testing.T) { + m := map[string]interface{}{"key": "string"} + result := GetFloat64(m, "key") + assert.Equal(t, 0.0, result) + }) +} + +func TestGetInt(t *testing.T) { + t.Run("int64 value", func(t *testing.T) { + m := map[string]interface{}{"key": int64(42)} + result := GetInt(m, "key") + assert.Equal(t, 42, result) + }) + + t.Run("int value", func(t *testing.T) { + m := map[string]interface{}{"key": 42} + result := GetInt(m, "key") + assert.Equal(t, 42, result) + }) + + t.Run("float64 value", func(t *testing.T) { + m := map[string]interface{}{"key": 42.0} + result := GetInt(m, "key") + assert.Equal(t, 42, result) + }) + + t.Run("non-existing key", func(t *testing.T) { + m := map[string]interface{}{} + result := GetInt(m, "key") + assert.Equal(t, 0, result) + }) + + t.Run("non-numeric value", func(t *testing.T) { + m := map[string]interface{}{"key": "string"} + result := GetInt(m, "key") + assert.Equal(t, 0, result) + }) +} + +func TestGetBool(t *testing.T) { + t.Run("true value", func(t *testing.T) { + m := map[string]interface{}{"key": true} + result := GetBool(m, "key") + assert.True(t, result) + }) + + t.Run("false value", func(t *testing.T) { + m := map[string]interface{}{"key": false} + result := GetBool(m, "key") + assert.False(t, result) + }) + + t.Run("non-existing key", func(t *testing.T) { + m := map[string]interface{}{} + result := GetBool(m, "key") + assert.False(t, result) + }) + + t.Run("non-bool value", func(t *testing.T) { + m := map[string]interface{}{"key": "string"} + result := GetBool(m, "key") + assert.False(t, result) + }) +} + +func TestGetTime(t *testing.T) { + testTime := time.Now() + + t.Run("time.Time value", func(t *testing.T) { + m := map[string]interface{}{"key": testTime} + result := GetTime(m, "key") + assert.Equal(t, testTime, result) + }) + + t.Run("RFC3339 string value", func(t *testing.T) { + timeStr := "2023-01-01T12:00:00Z" + m := map[string]interface{}{"key": timeStr} + result := GetTime(m, "key") + expected, _ := time.Parse(time.RFC3339, timeStr) + assert.Equal(t, expected, result) + }) + + t.Run("invalid string value", func(t *testing.T) { + m := map[string]interface{}{"key": "invalid"} + result := GetTime(m, "key") + assert.True(t, result.IsZero()) + }) + + t.Run("non-existing key", func(t *testing.T) { + m := map[string]interface{}{} + result := GetTime(m, "key") + assert.True(t, result.IsZero()) + }) + + t.Run("non-time value", func(t *testing.T) { + m := map[string]interface{}{"key": 123} + result := GetTime(m, "key") + assert.True(t, result.IsZero()) + }) +}