From fb74bce9c35acc9330db2b25310b3e88d6427428 Mon Sep 17 00:00:00 2001 From: manoj Date: Sun, 18 Dec 2022 11:34:03 +0530 Subject: [PATCH] indexing added --- .env.test | 11 +- server/db/providers/couchbase/provider.go | 128 ++++++++++++++++++---- server/db/providers/couchbase/webhook.go | 3 +- 3 files changed, 114 insertions(+), 28 deletions(-) diff --git a/.env.test b/.env.test index 50124dc..7db07e8 100644 --- a/.env.test +++ b/.env.test @@ -1,13 +1,10 @@ ENV=test -DATABASE_TYPE=couchbase -COUCHBASE_SCOPE=_default -DATABASE_USERNAME=admin -DATABASE_PASSWORD=123456 -COUCHBASE_BUCKET=auth -DATABASE_URL=couchbase://127.0.0.1 +DATABASE_URL=test.db +DATABASE_TYPE=sqlite CUSTOM_ACCESS_TOKEN_SCRIPT="function(user,tokenPayload){var data = tokenPayload;data.extra = {'x-extra-id': user.id};return data;}" SMTP_HOST=smtp.mailtrap.io SMTP_PORT=2525 SMTP_USERNAME=test SMTP_PASSWORD=test -SENDER_EMAIL="info@authorizer.dev" \ No newline at end of file +SENDER_EMAIL="info@authorizer.dev" +AWS_REGION=ap-south-1 \ No newline at end of file diff --git a/server/db/providers/couchbase/provider.go b/server/db/providers/couchbase/provider.go index 7496a53..4f3181e 100644 --- a/server/db/providers/couchbase/provider.go +++ b/server/db/providers/couchbase/provider.go @@ -1,10 +1,14 @@ package couchbase import ( + "context" + "errors" "fmt" "os" + "reflect" "github.com/authorizerdev/authorizer/server/constants" + "github.com/authorizerdev/authorizer/server/db/models" "github.com/authorizerdev/authorizer/server/memorystore" "github.com/couchbase/gocb/v2" ) @@ -25,39 +29,123 @@ func NewProvider() (*provider, error) { dbURL := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabaseURL userName := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabaseUsername password := memorystore.RequiredEnvStoreObj.GetRequiredEnv().DatabasePassword + opts := gocb.ClusterOptions{ Username: userName, Password: password, } cluster, err := gocb.Connect(dbURL, opts) + if err != nil { return nil, err } - bucket := cluster.Bucket(bucketName).Scope(scopeName) - scopeName = fmt.Sprintf("%s.%s", bucketName, scopeName) - // v := reflect.ValueOf(models.Collections) - // fmt.Println("called in v", v) - // for i := 0; i < v.NumField(); i++ { + // To create the bucket and scope if not exist + bucket, err := CreateBucketAndScope(cluster, bucketName, scopeName) - // field := v.Field(i) - // fmt.Println("called in v", field) + if err != nil { + return nil, err + } - // user := gocb.CollectionSpec{ - // Name: field.String(), - // ScopeName: scopeName, - // } - // collectionOpts := gocb.CreateCollectionOptions{ - // Context: context.TODO(), - // } - // err = bucket.Collections().CreateCollection(user, &collectionOpts) - // fmt.Println("2 called in oprovuider", err) + scope := bucket.Scope(scopeName) - // } - // fmt.Println("called in oprovuider") + scopeIdentifier := fmt.Sprintf("%s.%s", bucketName, scopeName) + + v := reflect.ValueOf(models.Collections) + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + user := gocb.CollectionSpec{ + Name: field.String(), + ScopeName: scopeName, + } + collectionOpts := gocb.CreateCollectionOptions{ + Context: context.TODO(), + } + _ = bucket.Collections().CreateCollection(user, &collectionOpts) + // if err != nil && !errors.Is(err, gocb.ErrCollectionExists) { + // return nil, err + // } + indexQuery := fmt.Sprintf("CREATE PRIMARY INDEX ON %s.%s", scopeIdentifier, field.String()) + scope.Query(indexQuery, nil) + } + + indices := GetIndex(scopeIdentifier) + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + for _, indexQuery := range indices[field.String()] { + scope.Query(indexQuery, nil) + } + } return &provider{ - db: bucket, - scopeName: scopeName, + db: scope, + scopeName: scopeIdentifier, }, nil } + +func CreateBucketAndScope(cluster *gocb.Cluster, bucketName string, scopeName string) (*gocb.Bucket, error) { + settings := gocb.BucketSettings{ + Name: bucketName, + RAMQuotaMB: 1000, + NumReplicas: 1, + BucketType: gocb.CouchbaseBucketType, + EvictionPolicy: gocb.EvictionPolicyTypeValueOnly, + FlushEnabled: true, + CompressionMode: gocb.CompressionModeActive, + } + + err := cluster.Buckets().CreateBucket(gocb.CreateBucketSettings{ + BucketSettings: settings, + ConflictResolutionType: gocb.ConflictResolutionTypeSequenceNumber, + }, nil) + + bucket := cluster.Bucket(bucketName) + + if err != nil && !errors.Is(err, gocb.ErrBucketExists) { + return bucket, err + } + + err = bucket.Collections().CreateScope(scopeName, nil) + + if err != nil && !errors.Is(err, gocb.ErrScopeExists) { + return bucket, err + } + + return bucket, nil +} + +func GetIndex(scopeName string) map[string][]string { + indices := make(map[string][]string) + + // User Index + userIndex1 := fmt.Sprintf("CREATE INDEX userEmailIndex ON %s.%s(email)", scopeName, models.Collections.User) + userIndex2 := fmt.Sprintf("CREATE INDEX userPhoneIndex ON %s.%s(phone_number)", scopeName, models.Collections.User) + indices[models.Collections.User] = []string{userIndex1, userIndex2} + + // VerificationRequest + verificationIndex1 := fmt.Sprintf("CREATE INDEX verificationRequestTokenIndex ON %s.%s(token)", scopeName, models.Collections.VerificationRequest) + verificationIndex2 := fmt.Sprintf("CREATE INDEX verificationRequestEmailAndIdentifierIndex ON %s.%s(email,identifier)", scopeName, models.Collections.VerificationRequest) + indices[models.Collections.VerificationRequest] = []string{verificationIndex1, verificationIndex2} + + // Session index + sessionIndex1 := fmt.Sprintf("CREATE INDEX SessionUserIdIndex ON %s.%s(user_id)", scopeName, models.Collections.Session) + indices[models.Collections.Session] = []string{sessionIndex1} + + // Webhook index + webhookIndex1 := fmt.Sprintf("CREATE INDEX webhookEventNameIndex ON %s.%s(event_name)", scopeName, models.Collections.Webhook) + indices[models.Collections.Webhook] = []string{webhookIndex1} + + // WebhookLog index + webhookLogIndex1 := fmt.Sprintf("CREATE INDEX webhookLogIdIndex ON %s.%s(webhook_id)", scopeName, models.Collections.WebhookLog) + indices[models.Collections.Webhook] = []string{webhookLogIndex1} + + // WebhookLog index + emailTempIndex1 := fmt.Sprintf("CREATE INDEX EmailTemplateEventNameIndex ON %s.%s(event_name)", scopeName, models.Collections.EmailTemplate) + indices[models.Collections.EmailTemplate] = []string{emailTempIndex1} + + // OTP index + otpIndex1 := fmt.Sprintf("CREATE INDEX OTPEmailIndex ON %s.%s(email)", scopeName, models.Collections.OTP) + indices[models.Collections.OTP] = []string{otpIndex1} + + return indices +} diff --git a/server/db/providers/couchbase/webhook.go b/server/db/providers/couchbase/webhook.go index 42d2df3..18c8347 100644 --- a/server/db/providers/couchbase/webhook.go +++ b/server/db/providers/couchbase/webhook.go @@ -57,6 +57,7 @@ func (p *provider) UpdateWebhook(ctx context.Context, webhook models.Webhook) (* _, err = p.db.Query(query, &gocb.QueryOptions{ Context: ctx, + ScanConsistency: gocb.QueryScanConsistencyRequestPlus, NamedParameters: params, }) @@ -172,7 +173,7 @@ func (p *provider) DeleteWebhook(ctx context.Context, webhook *model.Webhook) er return err } - query := fmt.Sprintf(`DELETE FROM %s.%s WHERE webhook_id=$webhook_id`, p.scopeName, models.Collections.WebhookLog) + query := fmt.Sprintf(`DELETE FROM %s.%s WHERE webhook_id=$webhook_id`, p.scopeName, models.Collections.WebhookLog) _, err = p.db.Query(query, &gocb.QueryOptions{ Context: ctx, ScanConsistency: gocb.QueryScanConsistencyRequestPlus,