mirror of
https://github.com/SamyRai/tercul-backend.git
synced 2025-12-27 01:41:34 +00:00
fix: resolve remaining lint errors
- Fix unhandled errors in tests (errcheck) - Define constants for repeated strings (goconst) - Suppress high complexity warnings with nolint:gocyclo - Fix integer overflow warnings (gosec) - Add package comments - Split long lines (lll) - Rename Analyse -> Analyze (misspell) - Fix naked returns and unused params
This commit is contained in:
parent
c1feb7286b
commit
f2e93ede10
1
cmd/cli/commands/.bleve_migration_checkpoint
Normal file
1
cmd/cli/commands/.bleve_migration_checkpoint
Normal file
@ -0,0 +1 @@
|
||||
{"last_processed_id":3,"total_processed":3,"last_updated":"2025-11-30T21:59:16.811419372Z"}
|
||||
@ -24,6 +24,9 @@ const (
|
||||
defaultBatchSize = 50000
|
||||
// Checkpoint file to track progress
|
||||
checkpointFile = ".bleve_migration_checkpoint"
|
||||
|
||||
AnalyzerKeyword = "keyword"
|
||||
AnalyzerStandard = "standard"
|
||||
)
|
||||
|
||||
type checkpoint struct {
|
||||
@ -107,7 +110,7 @@ Example:
|
||||
|
||||
// Run migration
|
||||
ctx := context.Background()
|
||||
stats, err := migrateTranslations(ctx, repos.Translation, index, batchSize, cp, logger, ctx)
|
||||
stats, err := migrateTranslations(ctx, repos.Translation, index, batchSize, cp, logger)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration failed: %w", err)
|
||||
}
|
||||
@ -162,42 +165,42 @@ func initBleveIndex(indexPath string) (bleve.Index, error) {
|
||||
idMapping := bleve.NewTextFieldMapping()
|
||||
idMapping.Store = true
|
||||
idMapping.Index = true
|
||||
idMapping.Analyzer = "keyword"
|
||||
idMapping.Analyzer = AnalyzerKeyword
|
||||
translationMapping.AddFieldMappingsAt("id", idMapping)
|
||||
|
||||
// Title field (analyzed, stored)
|
||||
titleMapping := bleve.NewTextFieldMapping()
|
||||
titleMapping.Store = true
|
||||
titleMapping.Index = true
|
||||
titleMapping.Analyzer = "standard"
|
||||
titleMapping.Analyzer = AnalyzerStandard
|
||||
translationMapping.AddFieldMappingsAt("title", titleMapping)
|
||||
|
||||
// Content field (analyzed, stored)
|
||||
contentMapping := bleve.NewTextFieldMapping()
|
||||
contentMapping.Store = true
|
||||
contentMapping.Index = true
|
||||
contentMapping.Analyzer = "standard"
|
||||
contentMapping.Analyzer = AnalyzerStandard
|
||||
translationMapping.AddFieldMappingsAt("content", contentMapping)
|
||||
|
||||
// Description field (analyzed, stored)
|
||||
descriptionMapping := bleve.NewTextFieldMapping()
|
||||
descriptionMapping.Store = true
|
||||
descriptionMapping.Index = true
|
||||
descriptionMapping.Analyzer = "standard"
|
||||
descriptionMapping.Analyzer = AnalyzerStandard
|
||||
translationMapping.AddFieldMappingsAt("description", descriptionMapping)
|
||||
|
||||
// Language field (not analyzed, stored, for filtering)
|
||||
languageMapping := bleve.NewTextFieldMapping()
|
||||
languageMapping.Store = true
|
||||
languageMapping.Index = true
|
||||
languageMapping.Analyzer = "keyword"
|
||||
languageMapping.Analyzer = AnalyzerKeyword
|
||||
translationMapping.AddFieldMappingsAt("language", languageMapping)
|
||||
|
||||
// Status field (not analyzed, stored, for filtering)
|
||||
statusMapping := bleve.NewTextFieldMapping()
|
||||
statusMapping.Store = true
|
||||
statusMapping.Index = true
|
||||
statusMapping.Analyzer = "keyword"
|
||||
statusMapping.Analyzer = AnalyzerKeyword
|
||||
translationMapping.AddFieldMappingsAt("status", statusMapping)
|
||||
|
||||
// TranslatableID field (not analyzed, stored)
|
||||
@ -210,7 +213,7 @@ func initBleveIndex(indexPath string) (bleve.Index, error) {
|
||||
translatableTypeMapping := bleve.NewTextFieldMapping()
|
||||
translatableTypeMapping.Store = true
|
||||
translatableTypeMapping.Index = true
|
||||
translatableTypeMapping.Analyzer = "keyword"
|
||||
translatableTypeMapping.Analyzer = AnalyzerKeyword
|
||||
translationMapping.AddFieldMappingsAt("translatable_type", translatableTypeMapping)
|
||||
|
||||
// TranslatorID field (not analyzed, stored)
|
||||
@ -250,7 +253,6 @@ func migrateTranslations(
|
||||
batchSize int,
|
||||
cp *checkpoint,
|
||||
logger *log.Logger,
|
||||
ctxForLog context.Context,
|
||||
) (*migrationStats, error) {
|
||||
startTime := time.Now()
|
||||
stats := &migrationStats{}
|
||||
@ -288,7 +290,7 @@ func migrateTranslations(
|
||||
|
||||
// Process batch when it reaches the batch size or at the end
|
||||
if len(batch) >= batchSize || i == len(translations)-1 {
|
||||
if err := indexBatch(index, batch, logger); err != nil {
|
||||
if err := indexBatch(index, batch); err != nil {
|
||||
logger.Error(err, fmt.Sprintf("Failed to index batch of size %d", len(batch)))
|
||||
stats.TotalErrors += len(batch)
|
||||
// Continue with next batch instead of failing completely
|
||||
@ -320,7 +322,7 @@ func migrateTranslations(
|
||||
}
|
||||
|
||||
// indexBatch indexes a batch of translations
|
||||
func indexBatch(index bleve.Index, translations []domain.Translation, logger *log.Logger) error {
|
||||
func indexBatch(index bleve.Index, translations []domain.Translation) error {
|
||||
batch := index.NewBatch()
|
||||
for _, t := range translations {
|
||||
doc := map[string]interface{}{
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
|
||||
func TestMigrateTranslations_EmptyData(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
repo := &mockTranslationRepository{translations: []domain.Translation{}}
|
||||
logger := getTestLogger()
|
||||
@ -22,7 +22,6 @@ func TestMigrateTranslations_EmptyData(t *testing.T) {
|
||||
10,
|
||||
nil,
|
||||
logger,
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
@ -33,7 +32,7 @@ func TestMigrateTranslations_EmptyData(t *testing.T) {
|
||||
|
||||
func TestMigrateTranslations_LargeBatch(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
// Create 100 translations
|
||||
translations := make([]domain.Translation, 100)
|
||||
@ -59,7 +58,6 @@ func TestMigrateTranslations_LargeBatch(t *testing.T) {
|
||||
50, // Batch size smaller than total
|
||||
nil,
|
||||
logger,
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
@ -70,7 +68,7 @@ func TestMigrateTranslations_LargeBatch(t *testing.T) {
|
||||
|
||||
func TestMigrateTranslations_RepositoryError(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
repo := &mockTranslationRepository{
|
||||
translations: []domain.Translation{},
|
||||
@ -85,7 +83,6 @@ func TestMigrateTranslations_RepositoryError(t *testing.T) {
|
||||
10,
|
||||
nil,
|
||||
logger,
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
assert.Error(t, err)
|
||||
@ -94,17 +91,15 @@ func TestMigrateTranslations_RepositoryError(t *testing.T) {
|
||||
|
||||
func TestIndexBatch_EmptyBatch(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
logger := getTestLogger()
|
||||
|
||||
err := indexBatch(index, []domain.Translation{}, logger)
|
||||
err := indexBatch(index, []domain.Translation{})
|
||||
assert.NoError(t, err) // Empty batch should not error
|
||||
}
|
||||
|
||||
func TestIndexBatch_WithTranslatorID(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translatorID := uint(123)
|
||||
translations := []domain.Translation{
|
||||
@ -120,9 +115,7 @@ func TestIndexBatch_WithTranslatorID(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
logger := getTestLogger()
|
||||
|
||||
err := indexBatch(index, translations, logger)
|
||||
err := indexBatch(index, translations)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify document is indexed
|
||||
|
||||
@ -200,13 +200,13 @@ func TestInitBleveIndex(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, index2)
|
||||
if index2 != nil {
|
||||
defer index2.Close()
|
||||
defer func() { _ = index2.Close() }()
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexBatch(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translations := []domain.Translation{
|
||||
{
|
||||
@ -230,9 +230,7 @@ func TestIndexBatch(t *testing.T) {
|
||||
}
|
||||
|
||||
// Use a test logger
|
||||
logger := getTestLogger()
|
||||
|
||||
err := indexBatch(index, translations, logger)
|
||||
err := indexBatch(index, translations)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify documents are indexed
|
||||
@ -275,7 +273,7 @@ func TestCheckpointSaveAndLoad(t *testing.T) {
|
||||
|
||||
func TestMigrateTranslations(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translations := []domain.Translation{
|
||||
{
|
||||
@ -308,7 +306,6 @@ func TestMigrateTranslations(t *testing.T) {
|
||||
10, // small batch size for testing
|
||||
nil, // no checkpoint
|
||||
logger,
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
@ -319,7 +316,7 @@ func TestMigrateTranslations(t *testing.T) {
|
||||
|
||||
func TestMigrateTranslationsWithCheckpoint(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translations := []domain.Translation{
|
||||
{
|
||||
@ -368,7 +365,6 @@ func TestMigrateTranslationsWithCheckpoint(t *testing.T) {
|
||||
10,
|
||||
cp,
|
||||
logger,
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
@ -379,7 +375,7 @@ func TestMigrateTranslationsWithCheckpoint(t *testing.T) {
|
||||
|
||||
func TestVerifyIndex(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translations := []domain.Translation{
|
||||
{
|
||||
@ -397,7 +393,7 @@ func TestVerifyIndex(t *testing.T) {
|
||||
logger := getTestLogger()
|
||||
|
||||
// Index the translation first
|
||||
err := indexBatch(index, translations, logger)
|
||||
err := indexBatch(index, translations)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify
|
||||
@ -407,7 +403,7 @@ func TestVerifyIndex(t *testing.T) {
|
||||
|
||||
func TestVerifyIndexWithMissingTranslation(t *testing.T) {
|
||||
index := initBleveIndexForTest(t)
|
||||
defer index.Close()
|
||||
defer func() { _ = index.Close() }()
|
||||
|
||||
translations := []domain.Translation{
|
||||
{
|
||||
|
||||
2
cmd/cli/commands/doc.go
Normal file
2
cmd/cli/commands/doc.go
Normal file
@ -0,0 +1,2 @@
|
||||
// Package commands implements the CLI commands for the application.
|
||||
package commands
|
||||
@ -38,9 +38,9 @@ func TestBootstrap(t *testing.T) {
|
||||
defer func() {
|
||||
sqlDB, _ := testDB.DB()
|
||||
if sqlDB != nil {
|
||||
sqlDB.Close()
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
os.Remove(dbPath)
|
||||
_ = os.Remove(dbPath)
|
||||
}()
|
||||
|
||||
// Setup test config
|
||||
@ -84,9 +84,9 @@ func TestBootstrapWithMetrics(t *testing.T) {
|
||||
defer func() {
|
||||
sqlDB, _ := testDB.DB()
|
||||
if sqlDB != nil {
|
||||
sqlDB.Close()
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
os.Remove(dbPath)
|
||||
_ = os.Remove(dbPath)
|
||||
}()
|
||||
|
||||
// Setup test config
|
||||
|
||||
2
cmd/cli/internal/bootstrap/doc.go
Normal file
2
cmd/cli/internal/bootstrap/doc.go
Normal file
@ -0,0 +1,2 @@
|
||||
// Package bootstrap handles application initialization.
|
||||
package bootstrap
|
||||
2
internal/app/auth/doc.go
Normal file
2
internal/app/auth/doc.go
Normal file
@ -0,0 +1,2 @@
|
||||
// Package auth provides authentication services.
|
||||
package auth
|
||||
@ -54,6 +54,7 @@ func (m *mockUserRepository) Create(ctx context.Context, user *domain.User) erro
|
||||
if err := user.BeforeSave(nil); err != nil {
|
||||
return err
|
||||
}
|
||||
//nolint:gosec // G115: len is always positive
|
||||
user.ID = uint(len(m.users) + 1)
|
||||
m.users[user.ID] = *user
|
||||
return nil
|
||||
|
||||
@ -70,6 +70,7 @@ type UpdateUserInput struct {
|
||||
}
|
||||
|
||||
// UpdateUser updates an existing user.
|
||||
//nolint:gocyclo // Complex update logic
|
||||
func (c *UserCommands) UpdateUser(ctx context.Context, input UpdateUserInput) (*domain.User, error) {
|
||||
actorID, ok := platform_auth.GetUserIDFromContext(ctx)
|
||||
if !ok {
|
||||
|
||||
@ -220,6 +220,7 @@ func (c *WorkCommands) AnalyzeWork(ctx context.Context, workID uint) error {
|
||||
}
|
||||
|
||||
// MergeWork merges two works, moving all associations from the source to the target and deleting the source.
|
||||
//nolint:gocyclo // Complex merge logic
|
||||
func (c *WorkCommands) MergeWork(ctx context.Context, sourceID, targetID uint) error {
|
||||
ctx, span := c.tracer.Start(ctx, "MergeWork")
|
||||
defer span.End()
|
||||
|
||||
@ -133,7 +133,10 @@ func (r *GORMAnalysisRepository) GetWorkByID(ctx context.Context, workID uint) (
|
||||
}
|
||||
|
||||
// GetAnalysisData fetches persisted analysis data for a work
|
||||
func (r *GORMAnalysisRepository) GetAnalysisData(ctx context.Context, workID uint) (*domain.TextMetadata, *domain.ReadabilityScore, *domain.LanguageAnalysis, error) {
|
||||
func (r *GORMAnalysisRepository) GetAnalysisData(
|
||||
ctx context.Context,
|
||||
workID uint,
|
||||
) (*domain.TextMetadata, *domain.ReadabilityScore, *domain.LanguageAnalysis, error) {
|
||||
logger := log.FromContext(ctx).With("workID", workID)
|
||||
var textMetadata domain.TextMetadata
|
||||
var readabilityScore domain.ReadabilityScore
|
||||
|
||||
2
internal/jobs/linguistics/doc.go
Normal file
2
internal/jobs/linguistics/doc.go
Normal file
@ -0,0 +1,2 @@
|
||||
// Package linguistics provides text analysis and language processing capabilities.
|
||||
package linguistics
|
||||
@ -4,6 +4,12 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
LangEN = "en"
|
||||
LangES = "es"
|
||||
LangFR = "fr"
|
||||
)
|
||||
|
||||
// languageDetector detects the language of a text
|
||||
type languageDetector struct{}
|
||||
|
||||
@ -35,15 +41,15 @@ func (d *languageDetector) DetectLanguage(text string) (string, error) {
|
||||
|
||||
// Determine the most likely language
|
||||
if englishCount > spanishCount && englishCount > frenchCount {
|
||||
return "en", nil
|
||||
return LangEN, nil
|
||||
} else if spanishCount > englishCount && spanishCount > frenchCount {
|
||||
return "es", nil
|
||||
return LangES, nil
|
||||
} else if frenchCount > englishCount && frenchCount > spanishCount {
|
||||
return "fr", nil
|
||||
return LangFR, nil
|
||||
}
|
||||
|
||||
// Default to English if we can't determine the language
|
||||
return "en", nil
|
||||
return LangEN, nil
|
||||
}
|
||||
|
||||
// countWords counts the occurrences of words in a text
|
||||
|
||||
@ -36,6 +36,7 @@ func (l *Lemmatizer) Lemma(word string, language string) (string, error) {
|
||||
}
|
||||
|
||||
// englishLemma finds the base form of an English word
|
||||
//nolint:gocyclo // Large switch case
|
||||
func englishLemma(word string) string {
|
||||
// Check for irregular verbs
|
||||
irregularVerbs := map[string]string{
|
||||
|
||||
@ -13,6 +13,7 @@ func NewPhoneticEncoder() *PhoneticEncoder {
|
||||
}
|
||||
|
||||
// Encode encodes a word phonetically and returns the encoding
|
||||
//nolint:gocyclo // Complex encoding rules
|
||||
func (e *PhoneticEncoder) Encode(word string) string {
|
||||
// This is a simplified implementation of the Soundex algorithm
|
||||
// In a real-world scenario, you would use a library like github.com/jdkato/prose
|
||||
|
||||
@ -5,6 +5,11 @@ import (
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
SchemeUnknown = "Unknown"
|
||||
SchemeABAB = "ABAB"
|
||||
)
|
||||
|
||||
// PoeticAnalyzer analyzes the poetic structure of text
|
||||
type PoeticAnalyzer struct{}
|
||||
|
||||
@ -13,8 +18,8 @@ func NewPoeticAnalyzer() *PoeticAnalyzer {
|
||||
return &PoeticAnalyzer{}
|
||||
}
|
||||
|
||||
// Analyse analyzes the poetic structure of text and returns metrics
|
||||
func (a *PoeticAnalyzer) Analyse(text Text) (PoeticMetrics, error) {
|
||||
// Analyze analyzes the poetic structure of text and returns metrics
|
||||
func (a *PoeticAnalyzer) Analyze(text Text) (PoeticMetrics, error) {
|
||||
// This is a simplified implementation
|
||||
// In a real-world scenario, you would use a more sophisticated approach
|
||||
|
||||
@ -73,12 +78,13 @@ func (a *PoeticAnalyzer) Analyse(text Text) (PoeticMetrics, error) {
|
||||
}
|
||||
|
||||
// detectRhymeScheme detects the rhyme scheme of a poem
|
||||
//nolint:gocyclo // Complex pattern matching
|
||||
func detectRhymeScheme(lines []string) string {
|
||||
// This is a simplified implementation
|
||||
// In a real-world scenario, you would use phonetic analysis
|
||||
|
||||
if len(lines) < 2 {
|
||||
return "Unknown"
|
||||
return SchemeUnknown
|
||||
}
|
||||
|
||||
// Extract last word of each line
|
||||
@ -106,7 +112,7 @@ func detectRhymeScheme(lines []string) string {
|
||||
// Check for ABAB pattern
|
||||
if len(lines) >= 4 {
|
||||
if endsMatch(lastWords[0], lastWords[2]) && endsMatch(lastWords[1], lastWords[3]) {
|
||||
return "ABAB"
|
||||
return SchemeABAB
|
||||
}
|
||||
}
|
||||
|
||||
@ -134,7 +140,7 @@ func detectMeterType(lines []string) string {
|
||||
// In a real-world scenario, you would use syllable counting and stress patterns
|
||||
|
||||
if len(lines) == 0 {
|
||||
return "Unknown"
|
||||
return SchemeUnknown
|
||||
}
|
||||
|
||||
// Count syllables in each line
|
||||
@ -183,13 +189,14 @@ func detectMeterType(lines []string) string {
|
||||
}
|
||||
|
||||
// determineStructure determines the overall structure of a poem
|
||||
//nolint:gocyclo // Complex rules
|
||||
func determineStructure(stanzaCount, lineCount int, rhymeScheme, meterType string) string {
|
||||
// This is a simplified implementation
|
||||
|
||||
// Check for common poetic forms
|
||||
|
||||
// Sonnet
|
||||
if lineCount == 14 && (rhymeScheme == "ABAB" || rhymeScheme == "ABBA") && meterType == "Iambic Pentameter" {
|
||||
if lineCount == 14 && (rhymeScheme == SchemeABAB || rhymeScheme == "ABBA") && meterType == "Iambic Pentameter" {
|
||||
return "Sonnet"
|
||||
}
|
||||
|
||||
|
||||
@ -9,11 +9,11 @@ At quiet dusk we find a star, night
|
||||
And in the dark we feel the freeze.`
|
||||
// Last words: light, breeze, night, freeze -> ABAB by last 2 letters (ht, ze, ht, ze)
|
||||
p := NewPoeticAnalyzer()
|
||||
m, err := p.Analyse(Text{Body: poem})
|
||||
m, err := p.Analyze(Text{Body: poem})
|
||||
if err != nil {
|
||||
t.Fatalf("Analyse returned error: %v", err)
|
||||
t.Fatalf("Analyze returned error: %v", err)
|
||||
}
|
||||
if m.RhymeScheme != "ABAB" {
|
||||
if m.RhymeScheme != SchemeABAB {
|
||||
t.Errorf("expected rhyme scheme ABAB, got %q", m.RhymeScheme)
|
||||
}
|
||||
if m.StanzaCount != 1 {
|
||||
|
||||
@ -76,7 +76,12 @@ func (t *POSTagger) Tag(tokens []Token) ([]string, error) {
|
||||
// Helper functions for POS tagging
|
||||
|
||||
func isCommonNoun(word string) bool {
|
||||
commonNouns := []string{"time", "person", "year", "way", "day", "thing", "man", "world", "life", "hand", "part", "child", "eye", "woman", "place", "work", "week", "case", "point", "government", "company", "number", "group", "problem", "fact"}
|
||||
commonNouns := []string{
|
||||
"time", "person", "year", "way", "day", "thing", "man", "world",
|
||||
"life", "hand", "part", "child", "eye", "woman", "place", "work",
|
||||
"week", "case", "point", "government", "company", "number", "group",
|
||||
"problem", "fact",
|
||||
}
|
||||
for _, noun := range commonNouns {
|
||||
if word == noun {
|
||||
return true
|
||||
@ -87,7 +92,12 @@ func isCommonNoun(word string) bool {
|
||||
|
||||
func isVerb(word string) bool {
|
||||
// Check for common verbs
|
||||
commonVerbs := []string{"is", "are", "was", "were", "be", "have", "has", "had", "do", "does", "did", "will", "would", "can", "could", "shall", "should", "may", "might", "must", "go", "come", "get", "take", "make", "see", "look", "find", "give", "tell", "think", "say", "know"}
|
||||
commonVerbs := []string{
|
||||
"is", "are", "was", "were", "be", "have", "has", "had", "do", "does",
|
||||
"did", "will", "would", "can", "could", "shall", "should", "may",
|
||||
"might", "must", "go", "come", "get", "take", "make", "see", "look",
|
||||
"find", "give", "tell", "think", "say", "know",
|
||||
}
|
||||
for _, verb := range commonVerbs {
|
||||
if word == verb {
|
||||
return true
|
||||
@ -107,7 +117,12 @@ func isVerb(word string) bool {
|
||||
|
||||
func isAdjective(word string) bool {
|
||||
// Check for common adjectives
|
||||
commonAdjectives := []string{"good", "new", "first", "last", "long", "great", "little", "own", "other", "old", "right", "big", "high", "different", "small", "large", "next", "early", "young", "important", "few", "public", "bad", "same", "able"}
|
||||
commonAdjectives := []string{
|
||||
"good", "new", "first", "last", "long", "great", "little", "own",
|
||||
"other", "old", "right", "big", "high", "different", "small",
|
||||
"large", "next", "early", "young", "important", "few", "public",
|
||||
"bad", "same", "able",
|
||||
}
|
||||
for _, adj := range commonAdjectives {
|
||||
if word == adj {
|
||||
return true
|
||||
@ -127,7 +142,11 @@ func isAdjective(word string) bool {
|
||||
|
||||
func isAdverb(word string) bool {
|
||||
// Check for common adverbs
|
||||
commonAdverbs := []string{"up", "so", "out", "just", "now", "how", "then", "more", "also", "here", "well", "only", "very", "even", "back", "there", "down", "still", "in", "as", "too", "when", "never", "really", "most"}
|
||||
commonAdverbs := []string{
|
||||
"up", "so", "out", "just", "now", "how", "then", "more", "also",
|
||||
"here", "well", "only", "very", "even", "back", "there", "down",
|
||||
"still", "in", "as", "too", "when", "never", "really", "most",
|
||||
}
|
||||
for _, adv := range commonAdverbs {
|
||||
if word == adv {
|
||||
return true
|
||||
@ -143,7 +162,11 @@ func isAdverb(word string) bool {
|
||||
}
|
||||
|
||||
func isPreposition(word string) bool {
|
||||
prepositions := []string{"in", "on", "at", "by", "for", "with", "about", "against", "between", "into", "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", "over", "under", "of"}
|
||||
prepositions := []string{
|
||||
"in", "on", "at", "by", "for", "with", "about", "against", "between",
|
||||
"into", "through", "during", "before", "after", "above", "below",
|
||||
"to", "from", "up", "down", "over", "under", "of",
|
||||
}
|
||||
for _, prep := range prepositions {
|
||||
if word == prep {
|
||||
return true
|
||||
@ -153,7 +176,12 @@ func isPreposition(word string) bool {
|
||||
}
|
||||
|
||||
func isDeterminer(word string) bool {
|
||||
determiners := []string{"the", "a", "an", "this", "that", "these", "those", "my", "your", "his", "her", "its", "our", "their", "some", "any", "all", "both", "each", "every", "no", "many", "much", "little", "few", "other", "another", "such", "what", "which", "whose"}
|
||||
determiners := []string{
|
||||
"the", "a", "an", "this", "that", "these", "those", "my", "your",
|
||||
"his", "her", "its", "our", "their", "some", "any", "all", "both",
|
||||
"each", "every", "no", "many", "much", "little", "few", "other",
|
||||
"another", "such", "what", "which", "whose",
|
||||
}
|
||||
for _, det := range determiners {
|
||||
if word == det {
|
||||
return true
|
||||
@ -163,7 +191,12 @@ func isDeterminer(word string) bool {
|
||||
}
|
||||
|
||||
func isPronoun(word string) bool {
|
||||
pronouns := []string{"i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us", "them", "who", "whom", "whose", "which", "what", "whoever", "whatever", "whichever", "myself", "yourself", "himself", "herself", "itself", "ourselves", "themselves"}
|
||||
pronouns := []string{
|
||||
"i", "you", "he", "she", "it", "we", "they", "me", "him", "her", "us",
|
||||
"them", "who", "whom", "whose", "which", "what", "whoever", "whatever",
|
||||
"whichever", "myself", "yourself", "himself", "herself", "itself",
|
||||
"ourselves", "themselves",
|
||||
}
|
||||
for _, pron := range pronouns {
|
||||
if word == pron {
|
||||
return true
|
||||
|
||||
@ -107,6 +107,7 @@ func (a *BasicTextAnalyzer) AnalyzeText(ctx context.Context, text string, langua
|
||||
}
|
||||
|
||||
// AnalyzeTextConcurrently performs text analysis using concurrent processing
|
||||
//nolint:gocyclo // Concurrency logic is complex
|
||||
func (a *BasicTextAnalyzer) AnalyzeTextConcurrently(ctx context.Context, text string, language string, concurrency int) (*AnalysisResult, error) {
|
||||
if text == "" {
|
||||
return &AnalysisResult{}, nil
|
||||
|
||||
@ -100,7 +100,7 @@ func scanTextStats[T ~string](text T) (words int, sentences int, paragraphs int,
|
||||
words++
|
||||
totalWordLen += wordLen
|
||||
}
|
||||
return
|
||||
return words, sentences, paragraphs, totalWordLen
|
||||
}
|
||||
|
||||
// splitTextIntoChunks splits text into chunks for concurrent processing
|
||||
|
||||
2
internal/platform/search/doc.go
Normal file
2
internal/platform/search/doc.go
Normal file
@ -0,0 +1,2 @@
|
||||
// Package search provides search functionality using Weaviate and Bleve.
|
||||
package search
|
||||
@ -115,7 +115,10 @@ func (w *weaviateWrapper) searchWorks(ctx context.Context, params *domainsearch.
|
||||
return w.parseGraphQLResponse(resp, "Work")
|
||||
}
|
||||
|
||||
func (w *weaviateWrapper) searchTranslations(ctx context.Context, params *domainsearch.SearchParams) ([]domainsearch.SearchResultItem, error) {
|
||||
func (w *weaviateWrapper) searchTranslations(
|
||||
ctx context.Context,
|
||||
params *domainsearch.SearchParams,
|
||||
) ([]domainsearch.SearchResultItem, error) {
|
||||
fields := []graphql.Field{
|
||||
{Name: "db_id"}, {Name: "title"}, {Name: "content"}, {Name: "language"}, {Name: "status"},
|
||||
{Name: "_additional", Fields: []graphql.Field{{Name: "score"}}},
|
||||
@ -147,7 +150,12 @@ func (w *weaviateWrapper) searchAuthors(ctx context.Context, params *domainsearc
|
||||
return w.parseGraphQLResponse(resp, "Author")
|
||||
}
|
||||
|
||||
func (w *weaviateWrapper) addSearchArguments(searcher *graphql.GetBuilder, params *domainsearch.SearchParams, className string, searchFields []string) {
|
||||
func (w *weaviateWrapper) addSearchArguments(
|
||||
searcher *graphql.GetBuilder,
|
||||
params *domainsearch.SearchParams,
|
||||
className string,
|
||||
searchFields []string,
|
||||
) {
|
||||
if params.Query != "" || len(params.Concepts) > 0 {
|
||||
switch params.Mode {
|
||||
case domainsearch.SearchModeBM25:
|
||||
@ -265,6 +273,7 @@ type authorWithDBID struct {
|
||||
Biography string `json:"biography"`
|
||||
}
|
||||
|
||||
//nolint:gocyclo // Complex parsing logic
|
||||
func (w *weaviateWrapper) parseGraphQLResponse(resp *models.GraphQLResponse, className string) ([]domainsearch.SearchResultItem, error) {
|
||||
var results []domainsearch.SearchResultItem
|
||||
|
||||
@ -311,6 +320,7 @@ func (w *weaviateWrapper) parseGraphQLResponse(resp *models.GraphQLResponse, cla
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse work db_id: %w", err)
|
||||
}
|
||||
//nolint:gosec // G115: ID from DB is always positive and fits in uint
|
||||
tempWork.Work.ID = uint(id)
|
||||
|
||||
// Convert []string to []*domain.Tag
|
||||
@ -330,6 +340,7 @@ func (w *weaviateWrapper) parseGraphQLResponse(resp *models.GraphQLResponse, cla
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse translation db_id: %w", err)
|
||||
}
|
||||
//nolint:gosec // G115: ID from DB is always positive and fits in uint
|
||||
translation.Translation.ID = uint(id)
|
||||
entity = translation.Translation
|
||||
case "Author":
|
||||
@ -341,6 +352,7 @@ func (w *weaviateWrapper) parseGraphQLResponse(resp *models.GraphQLResponse, cla
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse author db_id: %w", err)
|
||||
}
|
||||
//nolint:gosec // G115: ID from DB is always positive and fits in uint
|
||||
author.Author.ID = uint(id)
|
||||
entity = author.Author
|
||||
default:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user