Enhance configuration management and testing for backend

- Update .gitignore to selectively ignore pkg/ directories at the root level
- Modify CI workflow to verify all Go packages can be listed
- Introduce configuration management with a new config package, including loading environment variables
- Add comprehensive tests for configuration loading and environment variable handling
- Implement Neo4j database interaction functions with corresponding tests for data extraction

.gitea/workflows/ci.yml

@@ -87,7 +87,8 @@ jobs:
           pwd
           ls -la go.mod
           go list -m
-          go list ./pkg/config
+          # Verify all packages can be listed
+          go list ./...
         env:
           GO111MODULE: on
           GOPROXY: https://proxy.golang.org,direct

.gitignore

@@ -18,7 +18,10 @@ go.work
 # Build output
 bin/
 dist/
-pkg/
+# Note: pkg/ is used for source code in bugulma/backend, so we only ignore build artifact pkg/ directories
+# Ignore pkg/ only at root level or in specific build contexts, not in source code directories
+/pkg/
+*.a
 
 # Vendor directory
 vendor/

bugulma/backend/pkg/config/config.go

@@ -0,0 +1,80 @@
+package config
+
+import "os"
+
+type Config struct {
+	ServerPort string
+	JWTSecret  string
+	CORSOrigin string
+
+	// PostgreSQL configuration
+	PostgresHost     string
+	PostgresPort     string
+	PostgresUser     string
+	PostgresPassword string
+	PostgresDB       string
+	PostgresSSLMode  string
+
+	// Neo4j configuration
+	Neo4jURI      string
+	Neo4jUsername string
+	Neo4jPassword string
+	Neo4jDatabase string
+	Neo4jEnabled  bool
+
+	// Redis configuration
+	RedisURL string
+
+	// Ollama configuration
+	OllamaURL      string
+	OllamaModel    string
+	OllamaUsername string
+	OllamaPassword string
+
+	// Google Maps API configuration
+	GoogleMapsAPIKey      string
+	GoogleCloudProjectID  string
+}
+
+func Load() *Config {
+	return &Config{
+		ServerPort: getEnv("SERVER_PORT", "8080"),
+		JWTSecret:  getEnv("JWT_SECRET", "your-secret-key-change-in-production"),
+		CORSOrigin: getEnv("CORS_ORIGIN", "http://localhost:3000"),
+
+		// PostgreSQL defaults
+		PostgresHost:     getEnv("POSTGRES_HOST", "localhost"),
+		PostgresPort:     getEnv("POSTGRES_PORT", "5432"),
+		PostgresUser:     getEnv("POSTGRES_USER", "bugulma"),
+		PostgresPassword: getEnv("POSTGRES_PASSWORD", "bugulma"),
+		PostgresDB:       getEnv("POSTGRES_DB", "bugulma_city"),
+		PostgresSSLMode:  getEnv("POSTGRES_SSLMODE", "disable"),
+
+		// Neo4j defaults (disabled by default)
+		Neo4jURI:      getEnv("NEO4J_URI", "neo4j://localhost:7687"),
+		Neo4jUsername: getEnv("NEO4J_USERNAME", "neo4j"),
+		Neo4jPassword: getEnv("NEO4J_PASSWORD", "password"),
+		Neo4jDatabase: getEnv("NEO4J_DATABASE", "neo4j"),
+		Neo4jEnabled:  getEnv("NEO4J_ENABLED", "false") == "true",
+
+		// Redis defaults
+		RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
+
+		// Ollama defaults
+		OllamaURL:      getEnv("OLLAMA_URL", "http://localhost:11434"),
+		OllamaModel:    getEnv("OLLAMA_MODEL", "qwen2.5:7b"),
+		OllamaUsername: getEnv("OLLAMA_USERNAME", ""),
+		OllamaPassword: getEnv("OLLAMA_PASSWORD", ""),
+
+		// Google Maps API defaults
+		GoogleMapsAPIKey:     getEnv("GOOGLE_KG_API_KEY", ""), // Reuse KG API key if it works for Geocoding
+		GoogleCloudProjectID:  getEnv("GOOGLE_CLOUD_PROJECT_ID", ""),
+	}
+}
+
+func getEnv(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}

bugulma/backend/pkg/config/config_test.go

@@ -0,0 +1,104 @@
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetEnv(t *testing.T) {
+	// Test when environment variable is set
+	os.Setenv("TEST_VAR", "test_value")
+	defer os.Unsetenv("TEST_VAR")
+
+	result := getEnv("TEST_VAR", "fallback")
+	assert.Equal(t, "test_value", result)
+
+	// Test when environment variable is not set
+	result = getEnv("NON_EXISTENT_VAR", "fallback")
+	assert.Equal(t, "fallback", result)
+}
+
+func TestLoad(t *testing.T) {
+	// Clear any existing test environment variables
+	testVars := []string{
+		"SERVER_PORT", "JWT_SECRET", "CORS_ORIGIN",
+		"POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_DB", "POSTGRES_SSLMODE",
+		"NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "NEO4J_DATABASE", "NEO4J_ENABLED",
+	}
+
+	for _, v := range testVars {
+		os.Unsetenv(v)
+	}
+
+	// Test with all defaults
+	config := Load()
+
+	assert.Equal(t, "8080", config.ServerPort)
+	assert.Equal(t, "your-secret-key-change-in-production", config.JWTSecret)
+	assert.Equal(t, "http://localhost:3000", config.CORSOrigin)
+
+	assert.Equal(t, "localhost", config.PostgresHost)
+	assert.Equal(t, "5432", config.PostgresPort)
+	assert.Equal(t, "bugulma", config.PostgresUser)
+	assert.Equal(t, "bugulma", config.PostgresPassword)
+	assert.Equal(t, "bugulma_city", config.PostgresDB)
+	assert.Equal(t, "disable", config.PostgresSSLMode)
+
+	assert.Equal(t, "neo4j://localhost:7687", config.Neo4jURI)
+	assert.Equal(t, "neo4j", config.Neo4jUsername)
+	assert.Equal(t, "password", config.Neo4jPassword)
+	assert.Equal(t, "neo4j", config.Neo4jDatabase)
+	assert.False(t, config.Neo4jEnabled)
+}
+
+func TestLoad_WithEnvironmentVariables(t *testing.T) {
+	// Set test environment variables
+	os.Setenv("SERVER_PORT", "9090")
+	os.Setenv("JWT_SECRET", "test-secret")
+	os.Setenv("CORS_ORIGIN", "http://test.com")
+	os.Setenv("POSTGRES_HOST", "test-host")
+	os.Setenv("POSTGRES_PORT", "9999")
+	os.Setenv("POSTGRES_USER", "test-user")
+	os.Setenv("POSTGRES_PASSWORD", "test-pass")
+	os.Setenv("POSTGRES_DB", "test-db")
+	os.Setenv("POSTGRES_SSLMODE", "require")
+	os.Setenv("NEO4J_URI", "neo4j://test:7688")
+	os.Setenv("NEO4J_USERNAME", "test-user")
+	os.Setenv("NEO4J_PASSWORD", "test-pass")
+	os.Setenv("NEO4J_DATABASE", "test-db")
+	os.Setenv("NEO4J_ENABLED", "true")
+
+	defer func() {
+		// Clean up
+		testVars := []string{
+			"SERVER_PORT", "JWT_SECRET", "CORS_ORIGIN",
+			"POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_DB", "POSTGRES_SSLMODE",
+			"NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "NEO4J_DATABASE", "NEO4J_ENABLED",
+		}
+		for _, v := range testVars {
+			os.Unsetenv(v)
+		}
+	}()
+
+	// Test with environment variables set
+	config := Load()
+
+	assert.Equal(t, "9090", config.ServerPort)
+	assert.Equal(t, "test-secret", config.JWTSecret)
+	assert.Equal(t, "http://test.com", config.CORSOrigin)
+
+	assert.Equal(t, "test-host", config.PostgresHost)
+	assert.Equal(t, "9999", config.PostgresPort)
+	assert.Equal(t, "test-user", config.PostgresUser)
+	assert.Equal(t, "test-pass", config.PostgresPassword)
+	assert.Equal(t, "test-db", config.PostgresDB)
+	assert.Equal(t, "require", config.PostgresSSLMode)
+
+	assert.Equal(t, "neo4j://test:7688", config.Neo4jURI)
+	assert.Equal(t, "test-user", config.Neo4jUsername)
+	assert.Equal(t, "test-pass", config.Neo4jPassword)
+	assert.Equal(t, "test-db", config.Neo4jDatabase)
+	assert.True(t, config.Neo4jEnabled)
+}

bugulma/backend/pkg/database/neo4j.go

@@ -0,0 +1,205 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
+)
+
+// Neo4jConfig holds configuration for Neo4j connection
+type Neo4jConfig struct {
+	URI      string
+	Username string
+	Password string
+	Database string
+}
+
+// NewNeo4jDriver creates a new Neo4j driver instance
+func NewNeo4jDriver(config Neo4jConfig) (neo4j.DriverWithContext, error) {
+	driver, err := neo4j.NewDriverWithContext(
+		config.URI,
+		neo4j.BasicAuth(config.Username, config.Password, ""),
+		func(c *neo4j.Config) {
+			c.MaxConnectionPoolSize = 50
+			c.ConnectionAcquisitionTimeout = 30 * time.Second
+			c.MaxTransactionRetryTime = 30 * time.Second
+		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Neo4j driver: %w", err)
+	}
+
+	// Verify connectivity
+	ctx := context.Background()
+	if err := driver.VerifyConnectivity(ctx); err != nil {
+		driver.Close(ctx)
+		return nil, fmt.Errorf("failed to verify Neo4j connectivity: %w", err)
+	}
+
+	return driver, nil
+}
+
+// InitializeSchema creates constraints and indexes in Neo4j
+func InitializeSchema(ctx context.Context, driver neo4j.DriverWithContext, database string) error {
+	session := driver.NewSession(ctx, neo4j.SessionConfig{
+		AccessMode:   neo4j.AccessModeWrite,
+		DatabaseName: database,
+	})
+	defer session.Close(ctx)
+
+	// Constraints for uniqueness
+	constraints := []string{
+		"CREATE CONSTRAINT business_id_unique IF NOT EXISTS FOR (b:Business) REQUIRE b.id IS UNIQUE",
+		"CREATE CONSTRAINT organization_id_unique IF NOT EXISTS FOR (o:Organization) REQUIRE o.id IS UNIQUE",
+		"CREATE CONSTRAINT site_id_unique IF NOT EXISTS FOR (s:Site) REQUIRE s.id IS UNIQUE",
+		"CREATE CONSTRAINT resource_flow_id_unique IF NOT EXISTS FOR (rf:ResourceFlow) REQUIRE rf.id IS UNIQUE",
+		"CREATE CONSTRAINT match_id_unique IF NOT EXISTS FOR (m:Match) REQUIRE m.id IS UNIQUE",
+		"CREATE CONSTRAINT shared_asset_id_unique IF NOT EXISTS FOR (sa:SharedAsset) REQUIRE sa.id IS UNIQUE",
+	}
+
+	// Indexes for performance
+	indexes := []string{
+		"CREATE INDEX organization_name_index IF NOT EXISTS FOR (o:Organization) ON (o.name)",
+		"CREATE INDEX organization_sector_index IF NOT EXISTS FOR (o:Organization) ON (o.sector)",
+		"CREATE INDEX organization_subtype_index IF NOT EXISTS FOR (o:Organization) ON (o.subtype)",
+		"CREATE INDEX site_location_index IF NOT EXISTS FOR (s:Site) ON (s.latitude, s.longitude)",
+		"CREATE INDEX site_type_index IF NOT EXISTS FOR (s:Site) ON (s.site_type)",
+		"CREATE INDEX resource_flow_type_direction_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.type, rf.direction)",
+		"CREATE INDEX resource_flow_type_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.type)",
+		"CREATE INDEX resource_flow_direction_index IF NOT EXISTS FOR (rf:ResourceFlow) ON (rf.direction)",
+		"CREATE INDEX match_status_index IF NOT EXISTS FOR (m:Match) ON (m.status)",
+		"CREATE INDEX match_score_index IF NOT EXISTS FOR (m:Match) ON (m.compatibility_score)",
+		"CREATE INDEX shared_asset_type_index IF NOT EXISTS FOR (sa:SharedAsset) ON (sa.type)",
+	}
+
+	// Execute constraints
+	for _, constraint := range constraints {
+		if _, err := session.Run(ctx, constraint, nil); err != nil {
+			return fmt.Errorf("failed to create constraint: %w", err)
+		}
+	}
+
+	// Execute indexes
+	for _, index := range indexes {
+		if _, err := session.Run(ctx, index, nil); err != nil {
+			return fmt.Errorf("failed to create index: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// Helper functions for extracting values from Neo4j records
+
+// GetString extracts a string value from a map
+func GetString(m map[string]interface{}, key string) string {
+	if val, ok := m[key]; ok {
+		if str, ok := val.(string); ok {
+			return str
+		}
+	}
+	return ""
+}
+
+// GetFloat64 extracts a float64 value from a map
+func GetFloat64(m map[string]interface{}, key string) float64 {
+	if val, ok := m[key]; ok {
+		switch v := val.(type) {
+		case float64:
+			return v
+		case int64:
+			return float64(v)
+		case int:
+			return float64(v)
+		}
+	}
+	return 0
+}
+
+// GetInt extracts an int value from a map
+func GetInt(m map[string]interface{}, key string) int {
+	if val, ok := m[key]; ok {
+		switch v := val.(type) {
+		case int64:
+			return int(v)
+		case int:
+			return v
+		case float64:
+			return int(v)
+		}
+	}
+	return 0
+}
+
+// GetBool extracts a bool value from a map
+func GetBool(m map[string]interface{}, key string) bool {
+	if val, ok := m[key]; ok {
+		if b, ok := val.(bool); ok {
+			return b
+		}
+	}
+	return false
+}
+
+// GetTime extracts a time.Time value from a map
+func GetTime(m map[string]interface{}, key string) time.Time {
+	if val, ok := m[key]; ok {
+		if t, ok := val.(time.Time); ok {
+			return t
+		}
+		if str, ok := val.(string); ok {
+			if parsed, err := time.Parse(time.RFC3339, str); err == nil {
+				return parsed
+			}
+		}
+	}
+	return time.Time{}
+}
+
+// GetStringFromRecord extracts a string value from a Neo4j record
+func GetStringFromRecord(record neo4j.Record, key string) string {
+	val, ok := record.Get(key)
+	if !ok {
+		return ""
+	}
+	if str, ok := val.(string); ok {
+		return str
+	}
+	return ""
+}
+
+// GetFloat64FromRecord extracts a float64 value from a Neo4j record
+func GetFloat64FromRecord(record neo4j.Record, key string) float64 {
+	val, ok := record.Get(key)
+	if !ok {
+		return 0
+	}
+	switch v := val.(type) {
+	case float64:
+		return v
+	case int64:
+		return float64(v)
+	case int:
+		return float64(v)
+	}
+	return 0
+}
+
+// GetIntFromRecord extracts an int value from a Neo4j record
+func GetIntFromRecord(record neo4j.Record, key string) int {
+	val, ok := record.Get(key)
+	if !ok {
+		return 0
+	}
+	switch v := val.(type) {
+	case int64:
+		return int(v)
+	case int:
+		return v
+	case float64:
+		return int(v)
+	}
+	return 0
+}

bugulma/backend/pkg/database/neo4j_test.go

@@ -0,0 +1,154 @@
+package database
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetString(t *testing.T) {
+	t.Run("existing string key", func(t *testing.T) {
+		m := map[string]interface{}{"key": "value"}
+		result := GetString(m, "key")
+		assert.Equal(t, "value", result)
+	})
+
+	t.Run("non-existing key", func(t *testing.T) {
+		m := map[string]interface{}{}
+		result := GetString(m, "key")
+		assert.Equal(t, "", result)
+	})
+
+	t.Run("non-string value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 123}
+		result := GetString(m, "key")
+		assert.Equal(t, "", result)
+	})
+}
+
+func TestGetFloat64(t *testing.T) {
+	t.Run("float64 value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 3.14}
+		result := GetFloat64(m, "key")
+		assert.Equal(t, 3.14, result)
+	})
+
+	t.Run("int64 value", func(t *testing.T) {
+		m := map[string]interface{}{"key": int64(42)}
+		result := GetFloat64(m, "key")
+		assert.Equal(t, 42.0, result)
+	})
+
+	t.Run("int value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 42}
+		result := GetFloat64(m, "key")
+		assert.Equal(t, 42.0, result)
+	})
+
+	t.Run("non-existing key", func(t *testing.T) {
+		m := map[string]interface{}{}
+		result := GetFloat64(m, "key")
+		assert.Equal(t, 0.0, result)
+	})
+
+	t.Run("non-numeric value", func(t *testing.T) {
+		m := map[string]interface{}{"key": "string"}
+		result := GetFloat64(m, "key")
+		assert.Equal(t, 0.0, result)
+	})
+}
+
+func TestGetInt(t *testing.T) {
+	t.Run("int64 value", func(t *testing.T) {
+		m := map[string]interface{}{"key": int64(42)}
+		result := GetInt(m, "key")
+		assert.Equal(t, 42, result)
+	})
+
+	t.Run("int value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 42}
+		result := GetInt(m, "key")
+		assert.Equal(t, 42, result)
+	})
+
+	t.Run("float64 value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 42.0}
+		result := GetInt(m, "key")
+		assert.Equal(t, 42, result)
+	})
+
+	t.Run("non-existing key", func(t *testing.T) {
+		m := map[string]interface{}{}
+		result := GetInt(m, "key")
+		assert.Equal(t, 0, result)
+	})
+
+	t.Run("non-numeric value", func(t *testing.T) {
+		m := map[string]interface{}{"key": "string"}
+		result := GetInt(m, "key")
+		assert.Equal(t, 0, result)
+	})
+}
+
+func TestGetBool(t *testing.T) {
+	t.Run("true value", func(t *testing.T) {
+		m := map[string]interface{}{"key": true}
+		result := GetBool(m, "key")
+		assert.True(t, result)
+	})
+
+	t.Run("false value", func(t *testing.T) {
+		m := map[string]interface{}{"key": false}
+		result := GetBool(m, "key")
+		assert.False(t, result)
+	})
+
+	t.Run("non-existing key", func(t *testing.T) {
+		m := map[string]interface{}{}
+		result := GetBool(m, "key")
+		assert.False(t, result)
+	})
+
+	t.Run("non-bool value", func(t *testing.T) {
+		m := map[string]interface{}{"key": "string"}
+		result := GetBool(m, "key")
+		assert.False(t, result)
+	})
+}
+
+func TestGetTime(t *testing.T) {
+	testTime := time.Now()
+
+	t.Run("time.Time value", func(t *testing.T) {
+		m := map[string]interface{}{"key": testTime}
+		result := GetTime(m, "key")
+		assert.Equal(t, testTime, result)
+	})
+
+	t.Run("RFC3339 string value", func(t *testing.T) {
+		timeStr := "2023-01-01T12:00:00Z"
+		m := map[string]interface{}{"key": timeStr}
+		result := GetTime(m, "key")
+		expected, _ := time.Parse(time.RFC3339, timeStr)
+		assert.Equal(t, expected, result)
+	})
+
+	t.Run("invalid string value", func(t *testing.T) {
+		m := map[string]interface{}{"key": "invalid"}
+		result := GetTime(m, "key")
+		assert.True(t, result.IsZero())
+	})
+
+	t.Run("non-existing key", func(t *testing.T) {
+		m := map[string]interface{}{}
+		result := GetTime(m, "key")
+		assert.True(t, result.IsZero())
+	})
+
+	t.Run("non-time value", func(t *testing.T) {
+		m := map[string]interface{}{"key": 123}
+		result := GetTime(m, "key")
+		assert.True(t, result.IsZero())
+	})
+}