From c16d864ff437775d4bd2b9a11afbf776c631c061 Mon Sep 17 00:00:00 2001 From: TeoV Date: Tue, 9 Jan 2018 09:47:00 +0200 Subject: [PATCH] Update Contexts and indexing for AttributeProfile --- apier/v1/attributes.go | 4 +- apier/v1/attributes_it_test.go | 8 +- apier/v1/filter_indexes_it_test.go | 5 +- apier/v1/tp_it_test.go | 2 +- .../mysql/create_tariffplan_tables.sql | 2 +- .../postgres/create_tariffplan_tables.sql | 2 +- data/tariffplans/tutorial/Attributes.csv | 2 +- engine/attributes_it_test.go | 8 +- engine/attributes_test.go | 4 +- engine/datamanager.go | 62 ++++- engine/libattributes.go | 8 +- engine/loader_csv_test.go | 6 +- engine/model_helpers.go | 19 +- engine/model_helpers_test.go | 12 +- engine/models.go | 2 +- engine/onstor_it_test.go | 250 +++++++++++++----- engine/tp_reader.go | 38 +-- utils/apitpdata.go | 2 +- 18 files changed, 300 insertions(+), 136 deletions(-) diff --git a/apier/v1/attributes.go b/apier/v1/attributes.go index 2c604ab2f..df67e4f39 100644 --- a/apier/v1/attributes.go +++ b/apier/v1/attributes.go @@ -55,11 +55,11 @@ func (apierV1 *ApierV1) SetAttributeProfile(extAls *engine.ExternalAttributeProf } //RemAttributeProfile remove a specific Attribute Profile -func (apierV1 *ApierV1) RemAttributeProfile(arg utils.TenantID, reply *string) error { +func (apierV1 *ApierV1) RemAttributeProfile(arg utils.TenantID, contexts []string, reply *string) error { if missing := utils.MissingStructFields(&arg, []string{"Tenant", "ID"}); len(missing) != 0 { //Params missing return utils.NewErrMandatoryIeMissing(missing...) } - if err := apierV1.DataManager.RemoveAttributeProfile(arg.Tenant, arg.ID, utils.NonTransactional, true); err != nil { + if err := apierV1.DataManager.RemoveAttributeProfile(arg.Tenant, arg.ID, contexts, utils.NonTransactional, true); err != nil { if err.Error() != utils.ErrNotFound.Error() { err = utils.NewErrServerError(err) } diff --git a/apier/v1/attributes_it_test.go b/apier/v1/attributes_it_test.go index 1965657f2..badc376be 100644 --- a/apier/v1/attributes_it_test.go +++ b/apier/v1/attributes_it_test.go @@ -150,7 +150,7 @@ func testAttributeSGetAttributeForEvent(t *testing.T) { Tenant: ev.Tenant, ID: "ATTR_1", FilterIDs: []string{"FLTR_ACNT_1007"}, - Context: utils.MetaRating, + Contexts: []string{utils.MetaRating}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 1, 14, 0, 0, 0, 0, time.UTC)}, Attributes: []*engine.Attribute{ @@ -192,8 +192,8 @@ func testAttributeSGetAttributeForEvent(t *testing.T) { t.Errorf("Expecting: %s, received: %s", eAttrPrf.Tenant, attrReply.Tenant) } else if !reflect.DeepEqual(eAttrPrf.ID, attrReply.ID) { t.Errorf("Expecting: %s, received: %s", eAttrPrf.ID, attrReply.ID) - } else if !reflect.DeepEqual(eAttrPrf.Context, attrReply.Context) { - t.Errorf("Expecting: %s, received: %s", eAttrPrf.Tenant, attrReply.Tenant) + } else if !reflect.DeepEqual(eAttrPrf.Contexts, attrReply.Contexts) { + t.Errorf("Expecting: %s, received: %s", eAttrPrf.Contexts, attrReply.Contexts) } else if !reflect.DeepEqual(eAttrPrf.FilterIDs, attrReply.FilterIDs) { t.Errorf("Expecting: %s, received: %s", eAttrPrf.FilterIDs, attrReply.FilterIDs) } else if !reflect.DeepEqual(eAttrPrf.ActivationInterval.ActivationTime.Local(), attrReply.ActivationInterval.ActivationTime.Local()) { @@ -262,7 +262,7 @@ func testAttributeSSetAlsPrf(t *testing.T) { alsPrf = &engine.ExternalAttributeProfile{ Tenant: "cgrates.org", ID: "ApierTest", - Context: "*rating", + Contexts: []string{"*rating"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), diff --git a/apier/v1/filter_indexes_it_test.go b/apier/v1/filter_indexes_it_test.go index 494581845..7f6542196 100644 --- a/apier/v1/filter_indexes_it_test.go +++ b/apier/v1/filter_indexes_it_test.go @@ -1282,7 +1282,7 @@ func testV1FIdxSetAttributeProfileIndexes(t *testing.T) { alsPrf = &engine.ExternalAttributeProfile{ Tenant: "cgrates.org", ID: "ApierTest", - Context: "*rating", + Contexts: []string{"*rating1", "*rating2"}, FilterIDs: []string{"FLTR_1"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), @@ -1326,7 +1326,6 @@ func testV1FIdxSetAttributeProfileIndexes(t *testing.T) { nil); err != utils.ErrNotFound { t.Error(err) } - } func testV1FIdxComputeAttributeProfileIndexes(t *testing.T) { @@ -1395,7 +1394,7 @@ func testV1FIdxSetSecondAttributeProfileIndexes(t *testing.T) { alsPrf = &engine.ExternalAttributeProfile{ Tenant: "cgrates.org", ID: "ApierTest2", - Context: "*rating", + Contexts: []string{"*rating"}, FilterIDs: []string{"FLTR_2"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), diff --git a/apier/v1/tp_it_test.go b/apier/v1/tp_it_test.go index b349ca3b7..21dd82a32 100644 --- a/apier/v1/tp_it_test.go +++ b/apier/v1/tp_it_test.go @@ -131,7 +131,7 @@ func testTPExportTPToFolder(t *testing.T) { ExportPath: "/tmp/", ExportedFiles: []string{"RatingProfiles.csv", "CdrStats.csv", "Users.csv", "RatingPlans.csv", "Actions.csv", "AccountActions.csv", "Timings.csv", "SharedGroups.csv", "ActionPlans.csv", "ActionTriggers.cs", "DerivedChargers.csv", "Resources.csv", - "Stats.csv", "Thresholds.csv", "Destinations.csv", "Rates.csv", "DestinationRates.csv", "Filters.csv", "Suppliers.csv", "AliasProfiles.csv"}, + "Stats.csv", "Thresholds.csv", "Destinations.csv", "Rates.csv", "DestinationRates.csv", "Filters.csv", "Suppliers.csv", "Attributes.csv"}, } tpid := "TEST_TPID2" compress := true diff --git a/data/storage/mysql/create_tariffplan_tables.sql b/data/storage/mysql/create_tariffplan_tables.sql index 85b024f21..608e946c4 100644 --- a/data/storage/mysql/create_tariffplan_tables.sql +++ b/data/storage/mysql/create_tariffplan_tables.sql @@ -531,7 +531,7 @@ CREATE TABLE tp_attributes ( `tpid` varchar(64) NOT NULL, `tenant` varchar(64) NOT NULL, `id` varchar(64) NOT NULL, - `context` varchar(64) NOT NULL, + `contexts` varchar(64) NOT NULL, `filter_ids` varchar(64) NOT NULL, `activation_interval` varchar(64) NOT NULL, `field_name` varchar(64) NOT NULL, diff --git a/data/storage/postgres/create_tariffplan_tables.sql b/data/storage/postgres/create_tariffplan_tables.sql index 33db1f912..591198534 100644 --- a/data/storage/postgres/create_tariffplan_tables.sql +++ b/data/storage/postgres/create_tariffplan_tables.sql @@ -521,7 +521,7 @@ CREATE INDEX tp_suppliers_unique ON tp_suppliers ("tpid", "tenant", "id", "tpid" varchar(64) NOT NULL, "tenant"varchar(64) NOT NULL, "id" varchar(64) NOT NULL, - "context" varchar(64) NOT NULL, + "contexts" varchar(64) NOT NULL, "filter_ids" varchar(64) NOT NULL, "activation_interval" varchar(64) NOT NULL, "field_name" varchar(64) NOT NULL, diff --git a/data/tariffplans/tutorial/Attributes.csv b/data/tariffplans/tutorial/Attributes.csv index f48175d5b..f00bbb246 100644 --- a/data/tariffplans/tutorial/Attributes.csv +++ b/data/tariffplans/tutorial/Attributes.csv @@ -1,3 +1,3 @@ -#Tenant,ID,Context,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Weight +#Tenant,ID,Contexts,FilterIDs,ActivationInterval,FieldName,Initial,Substitute,Append,Weight cgrates.org,ATTR_1,*rating,FLTR_ACNT_1007,2014-01-14T00:00:00Z,Account,*any,1001,false,10 cgrates.org,ATTR_1,,,,Subject,*any,1001,true, diff --git a/engine/attributes_it_test.go b/engine/attributes_it_test.go index 798e350bc..05e52a640 100644 --- a/engine/attributes_it_test.go +++ b/engine/attributes_it_test.go @@ -30,7 +30,7 @@ func TestExternalAttributeProfileAsAttributeProfile(t *testing.T) { extAttr := &ExternalAttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), @@ -57,7 +57,7 @@ func TestExternalAttributeProfileAsAttributeProfile(t *testing.T) { expected := &AttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), @@ -85,7 +85,7 @@ func TestNewExternalAttributeProfileFromAttributeProfile(t *testing.T) { attrPrf := &AttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), @@ -98,7 +98,7 @@ func TestNewExternalAttributeProfileFromAttributeProfile(t *testing.T) { expected := &ExternalAttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC).Local(), diff --git a/engine/attributes_test.go b/engine/attributes_test.go index 6aa0b8aa2..a2d5af53f 100644 --- a/engine/attributes_test.go +++ b/engine/attributes_test.go @@ -64,7 +64,7 @@ func testPopulateAttrService(t *testing.T) { &AttributeProfile{ Tenant: "cgrates.org", ID: "attributeprofile1", - Context: context, + Contexts: []string{context}, FilterIDs: []string{"filter1"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), @@ -76,7 +76,7 @@ func testPopulateAttrService(t *testing.T) { &AttributeProfile{ Tenant: "cgrates.org", ID: "attributeprofile2", - Context: context, + Contexts: []string{context}, FilterIDs: []string{"filter2"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), diff --git a/engine/datamanager.go b/engine/datamanager.go index 33d05d507..ed87dcb75 100644 --- a/engine/datamanager.go +++ b/engine/datamanager.go @@ -1124,26 +1124,52 @@ func (dm *DataManager) GetAttributeProfile(tenant, id string, skipCache bool, tr } func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) (err error) { + oldAP, err := dm.GetAttributeProfile(ap.Tenant, ap.ID, true, utils.NonTransactional) + if err != nil && err != utils.ErrNotFound { + return err + } if err = dm.DataDB().SetAttributeProfileDrv(ap); err != nil { return err } //to be implemented in tests if withIndex { - indexer := NewReqFilterIndexer(dm, utils.AttributeProfilePrefix, ap.Tenant) - //remove old AttributeProfile indexes - if err = indexer.RemoveItemFromIndex(ap.ID); err != nil && - err.Error() != utils.ErrNotFound.Error() { - return - } - //Verify matching Filters for every FilterID from AttributeProfile - for _, fltrID := range ap.FilterIDs { - var fltr *Filter - if fltr, err = dm.GetFilter(ap.Tenant, fltrID, false, utils.NonTransactional); err != nil { - if err == utils.ErrNotFound { - err = fmt.Errorf("broken reference to filter: %+v for threshold: %+v", fltrID, ap) + if oldAP != nil { + for _, ctx := range oldAP.Contexts { + var needsRemove bool + if !utils.IsSliceMember(ap.Contexts, ctx) { + needsRemove = true + } else { + for _, fltrID := range oldAP.FilterIDs { + if !utils.IsSliceMember(ap.FilterIDs, fltrID) { + needsRemove = true + } + } } + if needsRemove { + if err = NewReqFilterIndexer(dm, utils.AttributeProfilePrefix, + utils.ConcatenatedKey(ap.Tenant, ctx)).RemoveItemFromIndex(ap.ID); err != nil { + return + } + } + } + } + for _, ctx := range ap.Contexts { + indexer := NewReqFilterIndexer(dm, utils.AttributeProfilePrefix, utils.ConcatenatedKey(ap.Tenant, ctx)) + //Verify matching Filters for every FilterID from AttributeProfile + for _, fltrID := range ap.FilterIDs { + var fltr *Filter + if fltr, err = dm.GetFilter(ap.Tenant, fltrID, false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for threshold: %+v", fltrID, ap) + } + return + } + indexer.IndexTPFilter(FilterToTPFilter(fltr), ap.ID) + } + if err = indexer.StoreIndexes(); err != nil { return } +<<<<<<< HEAD for _, flt := range fltr.RequestFilters { if flt.Type != MetaString { continue @@ -1158,19 +1184,27 @@ func (dm *DataManager) SetAttributeProfile(ap *AttributeProfile, withIndex bool) } if err = indexer.StoreIndexes(); err != nil { return +======= +>>>>>>> Update Contexts and indexing for AttributeProfile } } return } -func (dm *DataManager) RemoveAttributeProfile(tenant, id, transactionID string, withIndex bool) (err error) { +func (dm *DataManager) RemoveAttributeProfile(tenant, id string, contexts []string, + transactionID string, withIndex bool) (err error) { if err = dm.DataDB().RemoveAttributeProfileDrv(tenant, id); err != nil { return } cache.RemKey(utils.AttributeProfilePrefix+utils.ConcatenatedKey(tenant, id), cacheCommit(transactionID), transactionID) if withIndex { - return NewReqFilterIndexer(dm, utils.AttributeProfilePrefix, tenant).RemoveItemFromIndex(id) + for _, context := range contexts { + if err = NewReqFilterIndexer(dm, utils.AttributeProfilePrefix, + utils.ConcatenatedKey(tenant, context)).RemoveItemFromIndex(id); err != nil { + return + } + } } return } diff --git a/engine/libattributes.go b/engine/libattributes.go index d9192d7be..60dbae69f 100644 --- a/engine/libattributes.go +++ b/engine/libattributes.go @@ -34,7 +34,7 @@ type Attribute struct { type AttributeProfile struct { Tenant string ID string - Context string // bind this AttributeProfile to specific context + Contexts []string // bind this AttributeProfile to multiple contexts FilterIDs []string ActivationInterval *utils.ActivationInterval // Activation interval Attributes map[string]map[string]*Attribute // map[FieldName][InitialValue]*Attribute @@ -56,7 +56,7 @@ func (aps AttributeProfiles) Sort() { type ExternalAttributeProfile struct { Tenant string ID string - Context string // bind this AttributeProfile to specific context + Contexts []string // bind this AttributeProfile to multiple context FilterIDs []string ActivationInterval *utils.ActivationInterval // Activation interval Attributes []*Attribute @@ -67,7 +67,7 @@ func (eap *ExternalAttributeProfile) AsAttributeProfile() *AttributeProfile { alsPrf := &AttributeProfile{ Tenant: eap.Tenant, ID: eap.ID, - Context: eap.Context, + Contexts: eap.Contexts, FilterIDs: eap.FilterIDs, ActivationInterval: eap.ActivationInterval, Weight: eap.Weight, @@ -85,7 +85,7 @@ func NewExternalAttributeProfileFromAttributeProfile(alsPrf *AttributeProfile) * extals := &ExternalAttributeProfile{ Tenant: alsPrf.Tenant, ID: alsPrf.ID, - Context: alsPrf.Context, + Contexts: alsPrf.Contexts, ActivationInterval: alsPrf.ActivationInterval, FilterIDs: alsPrf.FilterIDs, Weight: alsPrf.Weight, diff --git a/engine/loader_csv_test.go b/engine/loader_csv_test.go index eee80a92c..b821bdd1b 100755 --- a/engine/loader_csv_test.go +++ b/engine/loader_csv_test.go @@ -1672,7 +1672,7 @@ func TestLoadAttributeProfiles(t *testing.T) { TPid: testTPID, Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_1"}, ActivationInterval: &utils.TPActivationInterval{ ActivationTime: "2014-07-29T15:00:00Z", @@ -1701,8 +1701,8 @@ func TestLoadAttributeProfiles(t *testing.T) { t.Errorf("Expecting: %+v, received: %+v", eAttrProfiles[resKey].Tenant, csvr.attributeProfiles[resKey].Tenant) } else if !reflect.DeepEqual(eAttrProfiles[resKey].ID, csvr.attributeProfiles[resKey].ID) { t.Errorf("Expecting: %+v, received: %+v", eAttrProfiles[resKey].ID, csvr.attributeProfiles[resKey].ID) - } else if !reflect.DeepEqual(eAttrProfiles[resKey].Context, csvr.attributeProfiles[resKey].Context) { - t.Errorf("Expecting: %+v, received: %+v", eAttrProfiles[resKey].Context, csvr.attributeProfiles[resKey].Context) + } else if !reflect.DeepEqual(eAttrProfiles[resKey].Contexts, csvr.attributeProfiles[resKey].Contexts) { + t.Errorf("Expecting: %+v, received: %+v", eAttrProfiles[resKey].Contexts, csvr.attributeProfiles[resKey].Contexts) } else if !reflect.DeepEqual(eAttrProfiles[resKey].FilterIDs, csvr.attributeProfiles[resKey].FilterIDs) { t.Errorf("Expecting: %+v, received: %+v", eAttrProfiles[resKey].FilterIDs, csvr.attributeProfiles[resKey].FilterIDs) } else if !reflect.DeepEqual(eAttrProfiles[resKey].ActivationInterval.ActivationTime, csvr.attributeProfiles[resKey].ActivationInterval.ActivationTime) { diff --git a/engine/model_helpers.go b/engine/model_helpers.go index dc7338e0f..9a835260b 100755 --- a/engine/model_helpers.go +++ b/engine/model_helpers.go @@ -2720,8 +2720,11 @@ func (tps TPAttributes) AsTPAttributes() (result []*utils.TPAttributeProfile) { th.FilterIDs = append(th.FilterIDs, filter) } } - if tp.Context != "" { - th.Context = tp.Context + if tp.Contexts != "" { + contextSplit := strings.Split(tp.Contexts, utils.INFIELD_SEP) + for _, context := range contextSplit { + th.Contexts = append(th.Contexts, context) + } } if tp.FieldName != "" { th.Attributes = append(th.Attributes, &utils.TPAttribute{ @@ -2761,8 +2764,11 @@ func APItoModelTPAttribute(th *utils.TPAttributeProfile) (mdls TPAttributes) { mdl.ActivationInterval += utils.INFIELD_SEP + th.ActivationInterval.ExpiryTime } } - if th.Context != "" { - mdl.Context = th.Context + for i, val := range th.Contexts { + if i != 0 { + mdl.Contexts += utils.INFIELD_SEP + } + mdl.Contexts += val } for i, val := range th.FilterIDs { if i != 0 { @@ -2789,12 +2795,15 @@ func APItoAttributeProfile(tpTH *utils.TPAttributeProfile, timezone string) (th ID: tpTH.ID, Weight: tpTH.Weight, FilterIDs: []string{}, - Context: tpTH.Context, + Contexts: []string{}, Attributes: make(map[string]map[string]*Attribute, len(tpTH.Attributes)), } for _, fli := range tpTH.FilterIDs { th.FilterIDs = append(th.FilterIDs, fli) } + for _, context := range tpTH.Contexts { + th.Contexts = append(th.Contexts, context) + } for _, reqAttr := range tpTH.Attributes { if _, has := th.Attributes[reqAttr.FieldName]; !has { th.Attributes[reqAttr.FieldName] = make(map[string]*Attribute) diff --git a/engine/model_helpers_test.go b/engine/model_helpers_test.go index 0bc55c1fe..b1f698bdb 100755 --- a/engine/model_helpers_test.go +++ b/engine/model_helpers_test.go @@ -1159,7 +1159,7 @@ func TestAPItoAttributeProfile(t *testing.T) { TPid: "TP1", Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.TPActivationInterval{ ActivationTime: "2014-07-14T14:35:00Z", @@ -1186,7 +1186,7 @@ func TestAPItoAttributeProfile(t *testing.T) { expected := &AttributeProfile{ Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 35, 0, 0, time.UTC), @@ -1206,7 +1206,7 @@ func TestAPItoModelTPAttribute(t *testing.T) { TPid: "TP1", Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.TPActivationInterval{ ActivationTime: "2014-07-14T14:35:00Z", @@ -1227,7 +1227,7 @@ func TestAPItoModelTPAttribute(t *testing.T) { Tpid: "TP1", Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: "con1", FilterIDs: "FLTR_ACNT_dan;FLTR_DST_DE", FieldName: "FL1", Initial: "In1", @@ -1249,7 +1249,7 @@ func TestModelAsTPAttribute(t *testing.T) { Tpid: "TP1", Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: "con1", FilterIDs: "FLTR_ACNT_dan;FLTR_DST_DE", FieldName: "FL1", Initial: "In1", @@ -1263,7 +1263,7 @@ func TestModelAsTPAttribute(t *testing.T) { TPid: "TP1", Tenant: "cgrates.org", ID: "ALS1", - Context: "con1", + Contexts: []string{"con1"}, FilterIDs: []string{"FLTR_ACNT_dan", "FLTR_DST_DE"}, ActivationInterval: &utils.TPActivationInterval{ ActivationTime: "2014-07-14T14:35:00Z", diff --git a/engine/models.go b/engine/models.go index c62c44074..30b26f525 100755 --- a/engine/models.go +++ b/engine/models.go @@ -547,7 +547,7 @@ type TPAttribute struct { Tpid string Tenant string `index:"0" re:""` ID string `index:"1" re:""` - Context string `index:"2" re:""` + Contexts string `index:"2" re:""` FilterIDs string `index:"3" re:""` ActivationInterval string `index:"4" re:""` FieldName string `index:"5" re:""` diff --git a/engine/onstor_it_test.go b/engine/onstor_it_test.go index 3a50a4bb2..f51514b81 100644 --- a/engine/onstor_it_test.go +++ b/engine/onstor_it_test.go @@ -43,69 +43,70 @@ var ( var sTestsOnStorIT = []func(t *testing.T){ testOnStorITFlush, testOnStorITIsDBEmpty, - testOnStorITSetGetDerivedCharges, - testOnStorITSetFilterIndexes, - testOnStorITGetFilterIndexes, - testOnStorITMatchFilterIndex, - testOnStorITCacheDestinations, - testOnStorITCacheReverseDestinations, - testOnStorITCacheRatingPlan, - testOnStorITCacheRatingProfile, - testOnStorITCacheActions, - testOnStorITCacheActionPlan, - testOnStorITCacheAccountActionPlans, - testOnStorITCacheActionTriggers, - testOnStorITCacheSharedGroup, - testOnStorITCacheDerivedChargers, - testOnStorITCacheLCR, - testOnStorITCacheAlias, - testOnStorITCacheReverseAlias, - testOnStorITCacheResource, - testOnStorITCacheResourceProfile, - testOnStorITCacheStatQueueProfile, - testOnStorITCacheStatQueue, - testOnStorITCacheThresholdProfile, - testOnStorITCacheThreshold, - testOnStorITCacheTiming, - testOnStorITCacheFilter, - testOnStorITCacheSupplierProfile, - testOnStorITCacheAttributeProfile, + // testOnStorITSetGetDerivedCharges, + // testOnStorITSetFilterIndexes, + // testOnStorITGetFilterIndexes, + // testOnStorITMatchFilterIndex, + // testOnStorITCacheDestinations, + // testOnStorITCacheReverseDestinations, + // testOnStorITCacheRatingPlan, + // testOnStorITCacheRatingProfile, + // testOnStorITCacheActions, + // testOnStorITCacheActionPlan, + // testOnStorITCacheAccountActionPlans, + // testOnStorITCacheActionTriggers, + // testOnStorITCacheSharedGroup, + // testOnStorITCacheDerivedChargers, + // testOnStorITCacheLCR, + // testOnStorITCacheAlias, + // testOnStorITCacheReverseAlias, + // testOnStorITCacheResource, + // testOnStorITCacheResourceProfile, + // testOnStorITCacheStatQueueProfile, + // testOnStorITCacheStatQueue, + // testOnStorITCacheThresholdProfile, + // testOnStorITCacheThreshold, + // testOnStorITCacheTiming, + // testOnStorITCacheFilter, + // testOnStorITCacheSupplierProfile, + // testOnStorITCacheAttributeProfile, // ToDo: test cache flush for a prefix // ToDo: testOnStorITLoadAccountingCache - testOnStorITHasData, - testOnStorITPushPop, - testOnStorITCRUDRatingPlan, - testOnStorITCRUDRatingProfile, - testOnStorITCRUDDestinations, - testOnStorITCRUDReverseDestinations, - testOnStorITCRUDLCR, - testOnStorITCRUDCdrStats, - testOnStorITCRUDActions, - testOnStorITCRUDSharedGroup, - testOnStorITCRUDActionTriggers, - testOnStorITCRUDActionPlan, - testOnStorITCRUDAccountActionPlans, - testOnStorITCRUDAccount, - testOnStorITCRUDCdrStatsQueue, - testOnStorITCRUDSubscribers, - testOnStorITCRUDUser, - testOnStorITCRUDAlias, - testOnStorITCRUDReverseAlias, - testOnStorITCRUDResource, - testOnStorITCRUDResourceProfile, - testOnStorITCRUDTiming, - testOnStorITCRUDHistory, - testOnStorITCRUDStructVersion, - testOnStorITCRUDStatQueueProfile, - testOnStorITCRUDStoredStatQueue, - testOnStorITCRUDThresholdProfile, - testOnStorITCRUDThreshold, - testOnStorITCRUDFilter, - testOnStorITCRUDSupplierProfile, - testOnStorITCRUDAttributeProfile, - testOnStorITFlush, - testOnStorITIsDBEmpty, - testOnStorITTestNewFilterIndexes, + // testOnStorITHasData, + // testOnStorITPushPop, + // testOnStorITCRUDRatingPlan, + // testOnStorITCRUDRatingProfile, + // testOnStorITCRUDDestinations, + // testOnStorITCRUDReverseDestinations, + // testOnStorITCRUDLCR, + // testOnStorITCRUDCdrStats, + // testOnStorITCRUDActions, + // testOnStorITCRUDSharedGroup, + // testOnStorITCRUDActionTriggers, + // testOnStorITCRUDActionPlan, + // testOnStorITCRUDAccountActionPlans, + // testOnStorITCRUDAccount, + // testOnStorITCRUDCdrStatsQueue, + // testOnStorITCRUDSubscribers, + // testOnStorITCRUDUser, + // testOnStorITCRUDAlias, + // testOnStorITCRUDReverseAlias, + // testOnStorITCRUDResource, + // testOnStorITCRUDResourceProfile, + // testOnStorITCRUDTiming, + // testOnStorITCRUDHistory, + // testOnStorITCRUDStructVersion, + // testOnStorITCRUDStatQueueProfile, + // testOnStorITCRUDStoredStatQueue, + // testOnStorITCRUDThresholdProfile, + // testOnStorITCRUDThreshold, + // testOnStorITCRUDFilter, + // testOnStorITCRUDSupplierProfile, + // testOnStorITCRUDAttributeProfile, + // testOnStorITFlush, + // testOnStorITIsDBEmpty, + // testOnStorITTestThresholdFilterIndexes, + testOnStorITTestAttributeProfileFilterIndexes, } func TestOnStorITRedisConnect(t *testing.T) { @@ -1232,7 +1233,7 @@ func testOnStorITCacheAttributeProfile(t *testing.T) { ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), }, - Context: "con1", + Contexts: []string{"con1"}, Attributes: mapSubstitutes, Weight: 20, } @@ -2622,7 +2623,7 @@ func testOnStorITCRUDAttributeProfile(t *testing.T) { ActivationInterval: &utils.ActivationInterval{ ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), }, - Context: "con1", + Contexts: []string{"con1"}, Attributes: mapSubstitutes, Weight: 20, } @@ -2642,7 +2643,7 @@ func testOnStorITCRUDAttributeProfile(t *testing.T) { } else if !reflect.DeepEqual(attrProfile, rcv) { t.Errorf("Expecting: %v, received: %v", attrProfile, rcv) } - if err := onStor.RemoveAttributeProfile(attrProfile.Tenant, attrProfile.ID, utils.NonTransactional, false); err != nil { + if err := onStor.RemoveAttributeProfile(attrProfile.Tenant, attrProfile.ID, attrProfile.Contexts, utils.NonTransactional, false); err != nil { t.Error(err) } if _, rcvErr := onStor.GetAttributeProfile("cgrates.org", "AttrPrf1", true, utils.NonTransactional); rcvErr != nil && rcvErr != utils.ErrNotFound { @@ -2650,7 +2651,7 @@ func testOnStorITCRUDAttributeProfile(t *testing.T) { } } -func testOnStorITTestNewFilterIndexes(t *testing.T) { +func testOnStorITTestThresholdFilterIndexes(t *testing.T) { fp := &Filter{ Tenant: "cgrates.org", ID: "Filter1", @@ -2875,5 +2876,124 @@ func testOnStorITTestNewFilterIndexes(t *testing.T) { t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) } } +} + +func testOnStorITTestAttributeProfileFilterIndexes(t *testing.T) { + fp := &Filter{ + Tenant: "cgrates.org", + ID: "Filter1", + RequestFilters: []*RequestFilter{ + &RequestFilter{ + FieldName: "EventType", + Type: "*string", + Values: []string{"Event1", "Event2"}, + }, + }, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + ExpiryTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + }, + } + if err := onStor.SetFilter(fp); err != nil { + t.Error(err) + } + mapSubstitutes := make(map[string]map[string]*Attribute) + mapSubstitutes["FN1"] = make(map[string]*Attribute) + mapSubstitutes["FN1"]["Init1"] = &Attribute{ + FieldName: "FN1", + Initial: "Init1", + Substitute: "Val1", + Append: true, + } + attrProfile := &AttributeProfile{ + Tenant: "cgrates.org", + ID: "ATTRPRF1", + FilterIDs: []string{"Filter1"}, + ActivationInterval: &utils.ActivationInterval{ + ActivationTime: time.Date(2014, 7, 14, 14, 25, 0, 0, time.UTC).Local(), + }, + Contexts: []string{"con1", "con2"}, + Attributes: mapSubstitutes, + Weight: 20, + } + //Set AttributeProfile with 2 contexts ( con1 , con2) + if err := onStor.SetAttributeProfile(attrProfile, true); err != nil { + t.Error(err) + } + eIdxes := map[string]utils.StringMap{ + "EventType:Event1": utils.StringMap{ + "ATTRPRF1": true, + }, + "EventType:Event2": utils.StringMap{ + "ATTRPRF1": true, + }, + } + reverseIdxes := map[string]utils.StringMap{ + "ATTRPRF1": utils.StringMap{ + "EventType:Event1": true, + "EventType:Event2": true, + }, + } + for _, ctx := range attrProfile.Contexts { + rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrProfile.Tenant, ctx)) + if rcvIdx, err := onStor.GetFilterIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), + nil); err != nil { + t.Error(err) + } else { + if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + } + if reverseRcvIdx, err := onStor.GetFilterReverseIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, true), + nil); err != nil { + t.Error(err) + } else { + if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { + t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) + } + } + } + //Set AttributeProfile with 1 new context (con3) + attrProfile.Contexts = []string{"con3"} + if err := onStor.SetAttributeProfile(attrProfile, true); err != nil { + t.Error(err) + } + //check indexes with the new context (con3) + rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrProfile.Tenant, "con3")) + if rcvIdx, err := onStor.GetFilterIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), + nil); err != nil { + t.Error(err) + } else { + if !reflect.DeepEqual(eIdxes, rcvIdx) { + t.Errorf("Expecting %+v, received: %+v", eIdxes, rcvIdx) + } + } + if reverseRcvIdx, err := onStor.GetFilterReverseIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, true), + nil); err != nil { + t.Error(err) + } else { + if !reflect.DeepEqual(reverseIdxes, reverseRcvIdx) { + t.Errorf("Expecting %+v, received: %+v", reverseIdxes, reverseRcvIdx) + } + } + + //check if old contexts was delete + for _, ctx := range []string{"con1", "con2"} { + rfi := NewReqFilterIndexer(onStor, utils.AttributeProfilePrefix, utils.ConcatenatedKey(attrProfile.Tenant, ctx)) + if _, err := onStor.GetFilterIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, false), + nil); err != nil && err != utils.ErrNotFound { + t.Error(err) + } + if _, err := onStor.GetFilterReverseIndexes( + GetDBIndexKey(rfi.itemType, rfi.dbKeySuffix, true), + nil); err != nil && err != utils.ErrNotFound { + t.Error(err) + } + } } diff --git a/engine/tp_reader.go b/engine/tp_reader.go index eb805a121..a45bf985b 100755 --- a/engine/tp_reader.go +++ b/engine/tp_reader.go @@ -1830,26 +1830,28 @@ func (tpr *TpReader) LoadAttributeProfilesFiltered(tag string) (err error) { tpr.attrTntID = append(tpr.attrTntID, &utils.TenantID{Tenant: tntID.Tenant, ID: tntID.ID}) } // index attribute profile for filters - attrKey := utils.ConcatenatedKey(tntID.Tenant, res.Context) - if _, has := tpr.attrIndexers[attrKey]; !has { - if tpr.attrIndexers[attrKey] = NewReqFilterIndexer(tpr.dm, utils.AttributeProfilePrefix, attrKey); err != nil { - return - } - } - for _, fltrID := range res.FilterIDs { - tpFltr, has := tpr.filters[utils.TenantID{Tenant: tntID.Tenant, ID: fltrID}] - if !has { - var fltr *Filter - if fltr, err = tpr.dm.GetFilter(tntID.Tenant, fltrID, false, utils.NonTransactional); err != nil { - if err == utils.ErrNotFound { - err = fmt.Errorf("broken reference to filter: %+v for resoruce: %+v", fltrID, res) - } + for _, context := range res.Contexts { + attrKey := utils.ConcatenatedKey(tntID.Tenant, context) + if _, has := tpr.attrIndexers[attrKey]; !has { + if tpr.attrIndexers[attrKey] = NewReqFilterIndexer(tpr.dm, utils.AttributeProfilePrefix, attrKey); err != nil { return - } else { - tpFltr = FilterToTPFilter(fltr) } - } else { - tpr.attrIndexers[attrKey].IndexTPFilter(tpFltr, res.ID) + } + for _, fltrID := range res.FilterIDs { + tpFltr, has := tpr.filters[utils.TenantID{Tenant: tntID.Tenant, ID: fltrID}] + if !has { + var fltr *Filter + if fltr, err = tpr.dm.GetFilter(tntID.Tenant, fltrID, false, utils.NonTransactional); err != nil { + if err == utils.ErrNotFound { + err = fmt.Errorf("broken reference to filter: %+v for resoruce: %+v", fltrID, res) + } + return + } else { + tpFltr = FilterToTPFilter(fltr) + } + } else { + tpr.attrIndexers[attrKey].IndexTPFilter(tpFltr, res.ID) + } } } } diff --git a/utils/apitpdata.go b/utils/apitpdata.go index e5d21c6ae..5bcaaa8f6 100755 --- a/utils/apitpdata.go +++ b/utils/apitpdata.go @@ -1408,7 +1408,7 @@ type TPAttributeProfile struct { ID string FilterIDs []string ActivationInterval *TPActivationInterval // Time when this limit becomes active and expires - Context string // bind this TPAttribute to specific context + Contexts []string // bind this TPAttribute to multiple context Attributes []*TPAttribute Weight float64 }